diff --git ql/src/test/results/clientpositive/auto_join14.q.out ql/src/test/results/clientpositive/auto_join14.q.out index cb27c0f..883350a 100644 --- ql/src/test/results/clientpositive/auto_join14.q.out +++ ql/src/test/results/clientpositive/auto_join14.q.out @@ -15,9 +15,6 @@ POSTHOOK: query: explain FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src)) (TOK_TABREF (TOK_TABNAME srcpart)) (and (AND (= (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL srcpart) key)) (= (. (TOK_TABLE_OR_COL srcpart) ds) '2008-04-08')) (> (. (TOK_TABLE_OR_COL src) key) 100)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL srcpart) value))))) - STAGE DEPENDENCIES: Stage-5 is a root stage Stage-4 depends on stages: Stage-5 @@ -35,52 +32,45 @@ STAGE PLANS: src TableScan alias: src + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: - expr: (key > 100) - type: boolean + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: 0 {key} 1 {value} - handleSkewJoin: false keys: - 0 [Column[key]] - 1 [Column[key]] - Position of Big Table: 1 + 0 key (type: string) + 1 key (type: string) Stage: Stage-4 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: - expr: (key > 100) - type: boolean + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 19 Data size: 3807 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: 0 {key} 1 {value} - handleSkewJoin: false keys: - 0 [Column[key]] - 1 [Column[key]] + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col5 - Position of Big Table: 1 + Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: UDFToInteger(_col0) - type: int - expr: _col5 - type: string + expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -102,7 +92,6 @@ STAGE PLANS: Stage: Stage-2 Stats-Aggr Operator - PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/combine2.q.out ql/src/test/results/clientpositive/combine2.q.out index 362e900..20bc062 100644 --- ql/src/test/results/clientpositive/combine2.q.out +++ ql/src/test/results/clientpositive/combine2.q.out @@ -102,9 +102,6 @@ POSTHOOK: Lineage: combine2 PARTITION(value=val_5).key EXPRESSION [(src)src.Fiel POSTHOOK: Lineage: combine2 PARTITION(value=val_8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: combine2 PARTITION(value=val_9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: combine2 PARTITION(value=|).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME combine2))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key))))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 is a root stage @@ -112,33 +109,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - combine2 + Map Operator Tree: TableScan alias: combine2 + Statistics: Num rows: 12 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 12 Data size: 14 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Statistics: Num rows: 12 Data size: 14 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Extract + Statistics: Num rows: 12 Data size: 14 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 + Statistics: Num rows: 12 Data size: 14 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -207,7 +196,27 @@ POSTHOOK: Lineage: combine2 PARTITION(value=val_8).key EXPRESSION [(src)src.Fiel POSTHOOK: Lineage: combine2 PARTITION(value=val_9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: combine2 PARTITION(value=|).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME combine2))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_WHERE (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL value))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + combine2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_FUNCTION + count + 1 + TOK_WHERE + TOK_FUNCTION + TOK_ISNOTNULL + TOK_TABLE_OR_COL + value + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -216,32 +225,23 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - combine2 + Map Operator Tree: TableScan alias: combine2 - Statistics: - numRows: 12 dataSize: 14 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 12 Data size: 14 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - Statistics: - numRows: 12 dataSize: 14 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 12 Data size: 14 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false + aggregations: count(1) mode: hash outputColumnNames: _col0 - Statistics: - numRows: 1 dataSize: 8 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: - numRows: 1 dataSize: 8 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: - expr: _col0 - type: bigint + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -585,27 +585,20 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false + aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: - numRows: 1 dataSize: 8 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: - expr: _col0 - type: bigint + expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: - numRows: 1 dataSize: 8 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 1 dataSize: 8 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -673,9 +666,6 @@ POSTHOOK: Lineage: combine2 PARTITION(value=val_5).key EXPRESSION [(src)src.Fiel POSTHOOK: Lineage: combine2 PARTITION(value=val_8).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: combine2 PARTITION(value=val_9).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: combine2 PARTITION(value=|).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL ds)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_WHERE (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL ds))) (TOK_GROUPBY (TOK_TABLE_OR_COL ds)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 is a root stage @@ -683,56 +673,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: COMPLETE Select Operator - expressions: - expr: ds - type: string + expressions: ds (type: string) outputColumnNames: ds + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: COMPLETE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: ds - type: string + aggregations: count(1) + keys: ds (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: COMPLETE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE File Output Operator compressed: false - GlobalTableId: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/ctas.q.out ql/src/test/results/clientpositive/ctas.q.out index 0566894..9668855 100644 --- ql/src/test/results/clientpositive/ctas.q.out +++ ql/src/test/results/clientpositive/ctas.q.out @@ -19,9 +19,6 @@ PREHOOK: query: explain create table nzhang_CTAS1 as select key k, value from sr PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 POSTHOOK: type: CREATETABLE_AS_SELECT -ABSTRACT SYNTAX TREE: - (TOK_CREATETABLE (TOK_TABNAME nzhang_CTAS1) TOK_LIKETABLE (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key) k) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))) (TOK_LIMIT 10)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -32,36 +29,27 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - src + Map Operator Tree: TableScan alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Extract + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -69,28 +57,22 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Extract + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -107,12 +89,9 @@ STAGE PLANS: Create Table Operator: Create Table columns: k string, value string - if not exists: false input format: org.apache.hadoop.mapred.TextInputFormat - # buckets: -1 output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat name: nzhang_CTAS1 - isExternal: false Stage: Stage-3 Stats-Aggr Operator @@ -180,9 +159,6 @@ PREHOOK: query: explain create table nzhang_ctas2 as select * from src sort by k PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain create table nzhang_ctas2 as select * from src sort by key, value limit 10 POSTHOOK: type: CREATETABLE_AS_SELECT -ABSTRACT SYNTAX TREE: - (TOK_CREATETABLE (TOK_TABNAME nzhang_ctas2) TOK_LIKETABLE (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))) (TOK_LIMIT 10)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -193,36 +169,27 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - src + Map Operator Tree: TableScan alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Extract + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -230,28 +197,22 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Extract + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -268,12 +229,9 @@ STAGE PLANS: Create Table Operator: Create Table columns: key string, value string - if not exists: false input format: org.apache.hadoop.mapred.TextInputFormat - # buckets: -1 output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat name: nzhang_ctas2 - isExternal: false Stage: Stage-3 Stats-Aggr Operator @@ -341,9 +299,6 @@ PREHOOK: query: explain create table nzhang_ctas3 row format serde "org.apache.h PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 POSTHOOK: type: CREATETABLE_AS_SELECT -ABSTRACT SYNTAX TREE: - (TOK_CREATETABLE (TOK_TABNAME nzhang_ctas3) TOK_LIKETABLE (TOK_TABLESERIALIZER (TOK_SERDENAME "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe")) TOK_TBLRCFILE (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (/ (TOK_TABLE_OR_COL key) 2) half_key) (TOK_SELEXPR (TOK_FUNCTION concat (TOK_TABLE_OR_COL value) "_con") conb)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL half_key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL conb))) (TOK_LIMIT 10)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -354,36 +309,27 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - src + Map Operator Tree: TableScan alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: (key / 2) - type: double - expr: concat(value, '_con') - type: string + expressions: (key / 2) (type: double), concat(value, '_con') (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: double - expr: _col1 - type: string + key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ - tag: -1 - value expressions: - expr: _col0 - type: double - expr: _col1 - type: string + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double), _col1 (type: string) Reduce Operator Tree: Extract + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -391,28 +337,22 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan Reduce Output Operator - key expressions: - expr: _col0 - type: double - expr: _col1 - type: string + key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ - tag: -1 - value expressions: - expr: _col0 - type: double - expr: _col1 - type: string + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double), _col1 (type: string) Reduce Operator Tree: Extract + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -429,13 +369,10 @@ STAGE PLANS: Create Table Operator: Create Table columns: half_key double, conb string - if not exists: false input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - # buckets: -1 output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde name: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: nzhang_ctas3 - isExternal: false Stage: Stage-3 Stats-Aggr Operator @@ -503,9 +440,6 @@ PREHOOK: query: explain create table if not exists nzhang_ctas3 as select key, v PREHOOK: type: CREATETABLE POSTHOOK: query: explain create table if not exists nzhang_ctas3 as select key, value from src sort by key, value limit 2 POSTHOOK: type: CREATETABLE -ABSTRACT SYNTAX TREE: - (TOK_CREATETABLE (TOK_TABNAME nzhang_ctas3) TOK_IFNOTEXISTS TOK_LIKETABLE (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))) (TOK_LIMIT 2)))) - STAGE DEPENDENCIES: STAGE PLANS: @@ -569,9 +503,6 @@ PREHOOK: query: explain create table nzhang_ctas4 row format delimited fields te PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 POSTHOOK: type: CREATETABLE_AS_SELECT -ABSTRACT SYNTAX TREE: - (TOK_CREATETABLE (TOK_TABNAME nzhang_ctas4) TOK_LIKETABLE (TOK_TABLEROWFORMAT (TOK_SERDEPROPS (TOK_TABLEROWFORMATFIELD ','))) TOK_TBLTEXTFILE (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))) (TOK_LIMIT 10)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -582,36 +513,27 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - src + Map Operator Tree: TableScan alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Extract + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -619,28 +541,22 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: Extract + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -658,12 +574,9 @@ STAGE PLANS: Create Table columns: key string, value string field delimiter: , - if not exists: false input format: org.apache.hadoop.mapred.TextInputFormat - # buckets: -1 output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat name: nzhang_ctas4 - isExternal: false Stage: Stage-3 Stats-Aggr Operator @@ -733,7 +646,44 @@ PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain extended create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '\012' stored as textfile as select key, value from src sort by key, value limit 10 POSTHOOK: type: CREATETABLE_AS_SELECT ABSTRACT SYNTAX TREE: - (TOK_CREATETABLE (TOK_TABNAME nzhang_ctas5) TOK_LIKETABLE (TOK_TABLEROWFORMAT (TOK_SERDEPROPS (TOK_TABLEROWFORMATFIELD ',') (TOK_TABLEROWFORMATLINES '\012'))) TOK_TBLTEXTFILE (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))) (TOK_LIMIT 10)))) + +TOK_CREATETABLE + TOK_TABNAME + nzhang_ctas5 + TOK_LIKETABLE + TOK_TABLEROWFORMAT + TOK_SERDEPROPS + TOK_TABLEROWFORMATFIELD + ',' + TOK_TABLEROWFORMATLINES + '\012' + TOK_TBLTEXTFILE + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SORTBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + TOK_LIMIT + 10 + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -745,37 +695,21 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - src + Map Operator Tree: TableScan alias: src - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + value expressions: _col0 (type: string), _col1 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -826,11 +760,10 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Limit - Statistics: - numRows: 10 dataSize: 2000 basicStatsState: COMPLETE colStatsState: NONE + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 @@ -851,25 +784,15 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan GatherStats: false Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Statistics: - numRows: 10 dataSize: 2000 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + value expressions: _col0 (type: string), _col1 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -898,18 +821,16 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 10 dataSize: 2000 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE Limit - Statistics: - numRows: 10 dataSize: 2000 basicStatsState: COMPLETE colStatsState: NONE + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 10 dataSize: 2000 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 10 Data size: 2000 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -940,14 +861,11 @@ STAGE PLANS: Create Table columns: key string, value string field delimiter: , - if not exists: false input format: org.apache.hadoop.mapred.TextInputFormat line delimiter: - # buckets: -1 output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat name: nzhang_ctas5 - isExternal: false Stage: Stage-3 Stats-Aggr Operator diff --git ql/src/test/results/clientpositive/input12.q.out ql/src/test/results/clientpositive/input12.q.out index 2bcb9a1..d65c121 100644 --- ql/src/test/results/clientpositive/input12.q.out +++ ql/src/test/results/clientpositive/input12.q.out @@ -29,9 +29,6 @@ INSERT OVERWRITE TABLE dest1 SELECT src.* WHERE src.key < 100 INSERT OVERWRITE TABLE dest2 SELECT src.key, src.value WHERE src.key >= 100 and src.key < 200 INSERT OVERWRITE TABLE dest3 PARTITION(ds='2008-04-08', hr='12') SELECT src.key WHERE src.key >= 200 POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME src)))) (TOK_WHERE (< (. (TOK_TABLE_OR_COL src) key) 100))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest2))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value))) (TOK_WHERE (and (>= (. (TOK_TABLE_OR_COL src) key) 100) (< (. (TOK_TABLE_OR_COL src) key) 200)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest3) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '12')))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key))) (TOK_WHERE (>= (. (TOK_TABLE_OR_COL src) key) 200)))) - STAGE DEPENDENCIES: Stage-3 is a root stage Stage-9 depends on stages: Stage-3 , consists of Stage-6, Stage-5, Stage-7 @@ -59,60 +56,50 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-3 Map Reduce - Alias -> Map Operator Tree: - src + Map Operator Tree: TableScan alias: src + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: - expr: (key < 100) - type: boolean + predicate: (key < 100) (type: boolean) + Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: UDFToInteger(key) - type: int - expr: value - type: string + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 Filter Operator - predicate: - expr: ((key >= 100) and (key < 200)) - type: boolean + predicate: ((key >= 100) and (key < 200)) (type: boolean) + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: UDFToInteger(key) - type: int - expr: value - type: string + expressions: UDFToInteger(key) (type: int), value (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 2 + Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2 Filter Operator - predicate: - expr: (key >= 200) - type: boolean + predicate: (key >= 200) (type: boolean) + Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: UDFToInteger(key) - type: int + expressions: UDFToInteger(key) (type: int) outputColumnNames: _col0 + Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 3 + Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -143,12 +130,10 @@ STAGE PLANS: Stage: Stage-5 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -157,12 +142,10 @@ STAGE PLANS: Stage: Stage-7 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -199,12 +182,10 @@ STAGE PLANS: Stage: Stage-11 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -213,12 +194,10 @@ STAGE PLANS: Stage: Stage-13 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -258,12 +237,10 @@ STAGE PLANS: Stage: Stage-17 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -272,12 +249,10 @@ STAGE PLANS: Stage: Stage-19 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -290,7 +265,6 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### - PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.* WHERE src.key < 100 INSERT OVERWRITE TABLE dest2 SELECT src.key, src.value WHERE src.key >= 100 and src.key < 200 diff --git ql/src/test/results/clientpositive/input39.q.out ql/src/test/results/clientpositive/input39.q.out index d1e3ab4..35d9418 100644 --- ql/src/test/results/clientpositive/input39.q.out +++ ql/src/test/results/clientpositive/input39.q.out @@ -68,9 +68,6 @@ POSTHOOK: Lineage: t1 PARTITION(ds=2).key SIMPLE [(src)src.FieldSchema(name:key, POSTHOOK: Lineage: t1 PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: t2 PARTITION(ds=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: t2 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME t1)) (TOK_TABREF (TOK_TABNAME t2)) (= (. (TOK_TABLE_OR_COL t1) key) (. (TOK_TABLE_OR_COL t2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_WHERE (and (= (. (TOK_TABLE_OR_COL t1) ds) '1') (= (. (TOK_TABLE_OR_COL t2) ds) '1'))))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -79,39 +76,29 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - t1 + Map Operator Tree: TableScan - alias: t1 + alias: t2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: - expr: (((hash(rand(460476415)) & 2147483647) % 32) = 0) - type: boolean + predicate: (((hash(rand(460476415)) & 2147483647) % 32) = 0) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - t2 + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan - alias: t2 + alias: t1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: - expr: (((hash(rand(460476415)) & 2147483647) % 32) = 0) - type: boolean + predicate: (((hash(rand(460476415)) & 2147483647) % 32) = 0) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -119,17 +106,16 @@ STAGE PLANS: condition expressions: 0 1 - handleSkewJoin: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false + aggregations: count(1) mode: hash outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -137,30 +123,25 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan Reduce Output Operator sort order: - tag: -1 - value expressions: - expr: _col0 - type: bigint + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false + aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: bigint + expressions: _col0 (type: bigint) outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -170,7 +151,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: select count(1) from t1 join t2 on t1.key=t2.key where t1.ds='1' and t2.ds='1' PREHOOK: type: QUERY PREHOOK: Input: default@t1 diff --git ql/src/test/results/clientpositive/join14.q.out ql/src/test/results/clientpositive/join14.q.out index 3f74a4f..d85c24e 100644 --- ql/src/test/results/clientpositive/join14.q.out +++ ql/src/test/results/clientpositive/join14.q.out @@ -15,9 +15,6 @@ POSTHOOK: query: EXPLAIN FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src)) (TOK_TABREF (TOK_TABNAME srcpart)) (and (AND (= (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL srcpart) key)) (= (. (TOK_TABLE_OR_COL srcpart) ds) '2008-04-08')) (> (. (TOK_TABLE_OR_COL src) key) 100)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL srcpart) value))))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -26,45 +23,31 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - src + Map Operator Tree: TableScan - alias: src + alias: srcpart + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: - expr: (key > 100) - type: boolean + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 19 Data size: 3807 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - srcpart + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 19 Data size: 3807 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) TableScan - alias: srcpart + alias: src + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: - expr: (key > 100) - type: boolean + predicate: (key > 100) (type: boolean) + Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: value - type: string + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 19 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) Reduce Operator Tree: Join Operator condition map: @@ -72,18 +55,15 @@ STAGE PLANS: condition expressions: 0 {VALUE._col0} 1 {VALUE._col1} - handleSkewJoin: false outputColumnNames: _col0, _col5 + Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: UDFToInteger(_col0) - type: int - expr: _col5 - type: string + expressions: UDFToInteger(_col0) (type: int), _col5 (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 20 Data size: 2093 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -103,7 +83,6 @@ STAGE PLANS: Stage: Stage-2 Stats-Aggr Operator - PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/list_bucket_dml_1.q.out ql/src/test/results/clientpositive/list_bucket_dml_1.q.out index ddfc09f..8a53191 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_1.q.out @@ -28,7 +28,39 @@ explain extended insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08' POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_dynamic_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL hr))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_dynamic_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + TOK_TABLE_OR_COL + hr + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -38,32 +70,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -199,7 +221,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.list_bucketing_dynamic_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -382,7 +403,39 @@ POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=11).va POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_dynamic_part))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL hr) '11')) (= (TOK_TABLE_OR_COL key) "484"))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_dynamic_part + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + = + TOK_TABLE_OR_COL + hr + '11' + = + TOK_TABLE_OR_COL + key + "484" + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -391,36 +444,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_dynamic_part + Map Operator Tree: TableScan alias: list_bucketing_dynamic_part - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (key = '484') - type: boolean - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + predicate: (key = '484') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_11.q.out ql/src/test/results/clientpositive/list_bucket_dml_11.q.out index 37f5cc1..18a9163 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_11.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_11.q.out @@ -36,7 +36,32 @@ insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', select key, value from src POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_static_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '11')))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_static_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + '11' + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -46,30 +71,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - src + Map Operator Tree: TableScan alias: src - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/hr=11/ - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -162,7 +179,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_static_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -245,7 +261,39 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_static_part))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL hr) '11')) (= (TOK_TABLE_OR_COL value) "val_466"))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_static_part + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + = + TOK_TABLE_OR_COL + hr + '11' + = + TOK_TABLE_OR_COL + value + "val_466" + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -254,36 +302,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_static_part + Map Operator Tree: TableScan alias: list_bucketing_static_part - Statistics: - numRows: 500 dataSize: 4812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (value = 'val_466') - type: boolean - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + predicate: (value = 'val_466') (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_12.q.out ql/src/test/results/clientpositive/list_bucket_dml_12.q.out index d11e176..899731f 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_12.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_12.q.out @@ -32,7 +32,38 @@ insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr select 1, key, 1, value, 1 from src POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_mul_col) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '11')))) (TOK_SELECT (TOK_SELEXPR 1) (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR 1)))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_mul_col + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + '11' + TOK_SELECT + TOK_SELEXPR + 1 + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + 1 + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + 1 + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -42,36 +73,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - src + Map Operator Tree: TableScan alias: src - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: UDFToString(1) - type: string - expr: key - type: string - expr: UDFToString(1) - type: string - expr: value - type: string - expr: UDFToString(1) - type: string + expressions: UDFToString(1) (type: string), key (type: string), UDFToString(1) (type: string), value (type: string), UDFToString(1) (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/hr=11/ - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -164,7 +181,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_mul_col -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -264,7 +280,53 @@ POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col3 EX POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col5 EXPRESSION [] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_mul_col))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (and (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL hr) '11')) (= (TOK_TABLE_OR_COL col2) "466")) (= (TOK_TABLE_OR_COL col4) "val_466"))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL col2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL col4)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL ds)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL hr))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_mul_col + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + and + and + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + = + TOK_TABLE_OR_COL + hr + '11' + = + TOK_TABLE_OR_COL + col2 + "466" + = + TOK_TABLE_OR_COL + col4 + "val_466" + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + col2 + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + col4 + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + ds + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + hr + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -273,68 +335,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_mul_col + Map Operator Tree: TableScan alias: list_bucketing_mul_col - Statistics: - numRows: 500 dataSize: 6312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 6312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((col2 = '466') and (col4 = 'val_466')) - type: boolean - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((col2 = '466') and (col4 = 'val_466')) (type: boolean) + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: col1 - type: string - expr: col2 - type: string - expr: col3 - type: string - expr: col4 - type: string - expr: col5 - type: string - expr: ds - type: string - expr: hr - type: string + expressions: col1 (type: string), col2 (type: string), col3 (type: string), col4 (type: string), col5 (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col1 - type: string - expr: _col3 - type: string - expr: _col5 - type: string - expr: _col6 - type: string + key expressions: _col1 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: string) sort order: ++++ - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col2 - type: string - expr: _col3 - type: string - expr: _col4 - type: string - expr: _col5 - type: string - expr: _col6 - type: string + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -385,15 +404,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -448,7 +465,53 @@ POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col3 EX POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col5 EXPRESSION [] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_mul_col))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (and (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL hr) '11')) (= (TOK_TABLE_OR_COL col2) "382")) (= (TOK_TABLE_OR_COL col4) "val_382"))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL col2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL col4)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL ds)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL hr))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_mul_col + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + and + and + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + = + TOK_TABLE_OR_COL + hr + '11' + = + TOK_TABLE_OR_COL + col2 + "382" + = + TOK_TABLE_OR_COL + col4 + "val_382" + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + col2 + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + col4 + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + ds + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + hr + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -457,68 +520,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_mul_col + Map Operator Tree: TableScan alias: list_bucketing_mul_col - Statistics: - numRows: 500 dataSize: 6312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 6312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((col2 = '382') and (col4 = 'val_382')) - type: boolean - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((col2 = '382') and (col4 = 'val_382')) (type: boolean) + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: col1 - type: string - expr: col2 - type: string - expr: col3 - type: string - expr: col4 - type: string - expr: col5 - type: string - expr: ds - type: string - expr: hr - type: string + expressions: col1 (type: string), col2 (type: string), col3 (type: string), col4 (type: string), col5 (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col1 - type: string - expr: _col3 - type: string - expr: _col5 - type: string - expr: _col6 - type: string + key expressions: _col1 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: string) sort order: ++++ - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col2 - type: string - expr: _col3 - type: string - expr: _col4 - type: string - expr: _col5 - type: string - expr: _col6 - type: string + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -569,15 +589,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_13.q.out ql/src/test/results/clientpositive/list_bucket_dml_13.q.out index 10694b1..199d4fe 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_13.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_13.q.out @@ -32,7 +32,38 @@ insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr select 1, key, 1, value, 1 from src POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_mul_col) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '2013-01-23+18:00:99')))) (TOK_SELECT (TOK_SELEXPR 1) (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR 1)))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_mul_col + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + '2013-01-23+18:00:99' + TOK_SELECT + TOK_SELEXPR + 1 + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + 1 + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + 1 + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -42,36 +73,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - src + Map Operator Tree: TableScan alias: src - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: UDFToString(1) - type: string - expr: key - type: string - expr: UDFToString(1) - type: string - expr: value - type: string - expr: UDFToString(1) - type: string + expressions: UDFToString(1) (type: string), key (type: string), UDFToString(1) (type: string), value (type: string), UDFToString(1) (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/ - Statistics: - numRows: 29 dataSize: 5812 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -164,7 +181,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_mul_col -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -264,7 +280,53 @@ POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+ POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col5 EXPRESSION [] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_mul_col))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (and (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL hr) '2013-01-23+18:00:99')) (= (TOK_TABLE_OR_COL col2) "466")) (= (TOK_TABLE_OR_COL col4) "val_466"))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL col2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL col4)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL ds)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL hr))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_mul_col + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + and + and + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + = + TOK_TABLE_OR_COL + hr + '2013-01-23+18:00:99' + = + TOK_TABLE_OR_COL + col2 + "466" + = + TOK_TABLE_OR_COL + col4 + "val_466" + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + col2 + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + col4 + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + ds + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + hr + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -273,68 +335,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_mul_col + Map Operator Tree: TableScan alias: list_bucketing_mul_col - Statistics: - numRows: 500 dataSize: 6312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 6312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((col2 = '466') and (col4 = 'val_466')) - type: boolean - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((col2 = '466') and (col4 = 'val_466')) (type: boolean) + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: col1 - type: string - expr: col2 - type: string - expr: col3 - type: string - expr: col4 - type: string - expr: col5 - type: string - expr: ds - type: string - expr: hr - type: string + expressions: col1 (type: string), col2 (type: string), col3 (type: string), col4 (type: string), col5 (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col1 - type: string - expr: _col3 - type: string - expr: _col5 - type: string - expr: _col6 - type: string + key expressions: _col1 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: string) sort order: ++++ - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col2 - type: string - expr: _col3 - type: string - expr: _col4 - type: string - expr: _col5 - type: string - expr: _col6 - type: string + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -385,15 +404,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 125 dataSize: 1578 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_2.q.out ql/src/test/results/clientpositive/list_bucket_dml_2.q.out index 41aad2b..96a6d47 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_2.q.out @@ -50,7 +50,37 @@ insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', select key, value from srcpart where ds = '2008-04-08' POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_static_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '11')))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_static_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + '11' + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -60,30 +90,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/hr=11/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -219,7 +241,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_static_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -334,7 +355,53 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_static_part))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (and (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL hr) '11')) (= (TOK_TABLE_OR_COL key) '484')) (= (TOK_TABLE_OR_COL value) 'val_484'))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL ds)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL hr))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_static_part + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + and + and + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + = + TOK_TABLE_OR_COL + hr + '11' + = + TOK_TABLE_OR_COL + key + '484' + = + TOK_TABLE_OR_COL + value + 'val_484' + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + ds + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + hr + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -343,56 +410,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_static_part + Map Operator Tree: TableScan alias: list_bucketing_static_part - Statistics: - numRows: 1000 dataSize: 9624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((key = '484') and (value = 'val_484')) - type: boolean - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((key = '484') and (value = 'val_484')) (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: ds - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col2 - type: string - expr: _col3 - type: string + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col2 - type: string - expr: _col3 - type: string + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -443,15 +479,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_3.q.out ql/src/test/results/clientpositive/list_bucket_dml_3.q.out index de5ed8e..7ffb860 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_3.q.out @@ -22,7 +22,37 @@ explain extended insert overwrite table list_bucketing_static_part partition (ds='2008-04-08', hr='11') select key, value from srcpart where ds='2008-04-08' POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_static_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '11')))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_static_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + '11' + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -32,30 +62,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/hr=11/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -191,7 +213,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.list_bucketing_static_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -310,7 +331,39 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_static_part))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL hr) '11')) (= (TOK_TABLE_OR_COL key) "484"))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_static_part + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + = + TOK_TABLE_OR_COL + hr + '11' + = + TOK_TABLE_OR_COL + key + "484" + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -319,36 +372,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_static_part + Map Operator Tree: TableScan alias: list_bucketing_static_part - Statistics: - numRows: 1000 dataSize: 10624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (key = '484') - type: boolean - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + predicate: (key = '484') (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_4.q.out ql/src/test/results/clientpositive/list_bucket_dml_4.q.out index 87804e0..3a46044 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_4.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_4.q.out @@ -58,7 +58,37 @@ insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', select key, value from srcpart where ds = '2008-04-08' POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_static_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '11')))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_static_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + '11' + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -68,30 +98,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/hr=11/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -227,7 +249,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_static_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -318,7 +339,37 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_static_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '11')))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_static_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + '11' + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -333,30 +384,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/hr=11/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -499,7 +542,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_static_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -710,7 +752,53 @@ POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).val POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_static_part))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (and (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL hr) '11')) (= (TOK_TABLE_OR_COL key) '484')) (= (TOK_TABLE_OR_COL value) 'val_484'))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL ds)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL hr))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_static_part + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + and + and + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + = + TOK_TABLE_OR_COL + hr + '11' + = + TOK_TABLE_OR_COL + key + '484' + = + TOK_TABLE_OR_COL + value + 'val_484' + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + ds + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + hr + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -719,56 +807,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_static_part + Map Operator Tree: TableScan alias: list_bucketing_static_part - Statistics: - numRows: 1000 dataSize: 9624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((key = '484') and (value = 'val_484')) - type: boolean - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((key = '484') and (value = 'val_484')) (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: ds - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col2 - type: string - expr: _col3 - type: string + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col2 - type: string - expr: _col3 - type: string + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -819,15 +876,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_5.q.out ql/src/test/results/clientpositive/list_bucket_dml_5.q.out index 0294842..6c086d7 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_5.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_5.q.out @@ -28,7 +28,39 @@ explain extended insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08' POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_dynamic_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL hr))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_dynamic_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + TOK_TABLE_OR_COL + hr + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -38,32 +70,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -199,7 +221,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.list_bucketing_dynamic_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -387,7 +408,58 @@ POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=11).va POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_dynamic_part))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_TABLE_OR_COL ds)) (TOK_SELEXPR (TOK_TABLE_OR_COL hr))) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL key) "103")) (= (TOK_TABLE_OR_COL value) "val_103"))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL ds)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL hr))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_dynamic_part + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + TOK_TABLE_OR_COL + ds + TOK_SELEXPR + TOK_TABLE_OR_COL + hr + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + = + TOK_TABLE_OR_COL + key + "103" + = + TOK_TABLE_OR_COL + value + "val_103" + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + ds + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + hr + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -396,56 +468,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_dynamic_part + Map Operator Tree: TableScan alias: list_bucketing_dynamic_part - Statistics: - numRows: 1000 dataSize: 10624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((key = '103') and (value = 'val_103')) - type: boolean - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((key = '103') and (value = 'val_103')) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: ds - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col2 - type: string - expr: _col3 - type: string + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col2 - type: string - expr: _col3 - type: string + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -539,15 +580,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_6.q.out ql/src/test/results/clientpositive/list_bucket_dml_6.q.out index af2da28..8738f15 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_6.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_6.q.out @@ -104,7 +104,47 @@ insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds = '2008-04-08' POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_dynamic_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION if (== (% (TOK_TABLE_OR_COL key) 100) 0) 'a1' 'b1'))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_dynamic_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + TOK_FUNCTION + if + == + % + TOK_TABLE_OR_COL + key + 100 + 0 + 'a1' + 'b1' + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -114,32 +154,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: if(((key % 100) = 0), 'a1', 'b1') - type: string + expressions: key (type: string), value (type: string), if(((key % 100) = 0), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -275,7 +305,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_dynamic_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -423,7 +452,47 @@ POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=a1).va POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_dynamic_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION if (== (% (TOK_TABLE_OR_COL key) 100) 0) 'a1' 'b1'))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_dynamic_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + TOK_FUNCTION + if + == + % + TOK_TABLE_OR_COL + key + 100 + 0 + 'a1' + 'b1' + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -438,32 +507,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: if(((key % 100) = 0), 'a1', 'b1') - type: string + expressions: key (type: string), value (type: string), if(((key % 100) = 0), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -608,7 +667,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_dynamic_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -902,7 +960,30 @@ POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).va POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_dynamic_part))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (= (TOK_TABLE_OR_COL key) '484') (= (TOK_TABLE_OR_COL value) 'val_484'))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_dynamic_part + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + = + TOK_TABLE_OR_COL + key + '484' + = + TOK_TABLE_OR_COL + value + 'val_484' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -911,40 +992,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_dynamic_part + Map Operator Tree: TableScan alias: list_bucketing_dynamic_part - Statistics: - numRows: 1000 dataSize: 9624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((key = '484') and (value = 'val_484')) - type: boolean - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((key = '484') and (value = 'val_484')) (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: ds - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_7.q.out ql/src/test/results/clientpositive/list_bucket_dml_7.q.out index 0696463..f8220ec 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_7.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_7.q.out @@ -52,7 +52,47 @@ insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds = '2008-04-08' POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_dynamic_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION if (== (% (TOK_TABLE_OR_COL key) 100) 0) 'a1' 'b1'))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_dynamic_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + TOK_FUNCTION + if + == + % + TOK_TABLE_OR_COL + key + 100 + 0 + 'a1' + 'b1' + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -62,32 +102,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: if(((key % 100) = 0), 'a1', 'b1') - type: string + expressions: key (type: string), value (type: string), if(((key % 100) = 0), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -223,7 +253,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_dynamic_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -371,7 +400,47 @@ POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=a1).va POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_dynamic_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION if (== (% (TOK_TABLE_OR_COL key) 100) 0) 'a1' 'b1'))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_dynamic_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + TOK_FUNCTION + if + == + % + TOK_TABLE_OR_COL + key + 100 + 0 + 'a1' + 'b1' + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -386,32 +455,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: if(((key % 100) = 0), 'a1', 'b1') - type: string + expressions: key (type: string), value (type: string), if(((key % 100) = 0), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -556,7 +615,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_dynamic_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -850,7 +908,30 @@ POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).va POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_dynamic_part))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (= (TOK_TABLE_OR_COL key) '484') (= (TOK_TABLE_OR_COL value) 'val_484'))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_dynamic_part + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + = + TOK_TABLE_OR_COL + key + '484' + = + TOK_TABLE_OR_COL + value + 'val_484' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -859,40 +940,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_dynamic_part + Map Operator Tree: TableScan alias: list_bucketing_dynamic_part - Statistics: - numRows: 1000 dataSize: 9624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((key = '484') and (value = 'val_484')) - type: boolean - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((key = '484') and (value = 'val_484')) (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: ds - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_8.q.out ql/src/test/results/clientpositive/list_bucket_dml_8.q.out index 7080f23..edb4e07 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_8.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_8.q.out @@ -108,7 +108,47 @@ insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds = '2008-04-08' POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_dynamic_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION if (== (% (TOK_TABLE_OR_COL key) 100) 0) 'a1' 'b1'))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_dynamic_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + TOK_FUNCTION + if + == + % + TOK_TABLE_OR_COL + key + 100 + 0 + 'a1' + 'b1' + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -118,32 +158,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: if(((key % 100) = 0), 'a1', 'b1') - type: string + expressions: key (type: string), value (type: string), if(((key % 100) = 0), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -279,7 +309,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_dynamic_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -520,7 +549,30 @@ POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=a1).va POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_dynamic_part))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (= (TOK_TABLE_OR_COL key) '484') (= (TOK_TABLE_OR_COL value) 'val_484'))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_dynamic_part + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + = + TOK_TABLE_OR_COL + key + '484' + = + TOK_TABLE_OR_COL + value + 'val_484' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -529,40 +581,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_dynamic_part + Map Operator Tree: TableScan alias: list_bucketing_dynamic_part - Statistics: - numRows: 1000 dataSize: 9624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 16 Data size: 136 Basic stats: PARTIAL Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((key = '484') and (value = 'val_484')) - type: boolean - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((key = '484') and (value = 'val_484')) (type: boolean) + Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: ds - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_9.q.out ql/src/test/results/clientpositive/list_bucket_dml_9.q.out index b6118ea..9d0eb29 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_9.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_9.q.out @@ -58,7 +58,37 @@ insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', select key, value from srcpart where ds = '2008-04-08' POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_static_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '11')))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_static_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + '11' + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -68,30 +98,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/hr=11/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -227,7 +249,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_static_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -318,7 +339,37 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME list_bucketing_static_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '11')))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + srcpart + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + list_bucketing_static_part + TOK_PARTSPEC + TOK_PARTVAL + ds + '2008-04-08' + TOK_PARTVAL + hr + '11' + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -333,30 +384,22 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - srcpart + Map Operator Tree: TableScan alias: srcpart - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/hr=11/ - Statistics: - numRows: 58 dataSize: 11624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat @@ -499,7 +542,6 @@ STAGE PLANS: #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_static_part -#### A masked pattern was here #### Stage: Stage-2 Stats-Aggr Operator @@ -710,7 +752,53 @@ POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).val POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME list_bucketing_static_part))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (and (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL hr) '11')) (= (TOK_TABLE_OR_COL key) '484')) (= (TOK_TABLE_OR_COL value) 'val_484'))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL ds)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL hr))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + list_bucketing_static_part + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + and + and + = + TOK_TABLE_OR_COL + ds + '2008-04-08' + = + TOK_TABLE_OR_COL + hr + '11' + = + TOK_TABLE_OR_COL + key + '484' + = + TOK_TABLE_OR_COL + value + 'val_484' + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + ds + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + hr + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -719,56 +807,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - list_bucketing_static_part + Map Operator Tree: TableScan alias: list_bucketing_static_part - Statistics: - numRows: 1000 dataSize: 9624 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((key = '484') and (value = 'val_484')) - type: boolean - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((key = '484') and (value = 'val_484')) (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: ds - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col2 - type: string - expr: _col3 - type: string + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) sort order: ++++ - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col2 - type: string - expr: _col3 - type: string + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -819,15 +876,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2406 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out index 24f9e99..bb5deee 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out @@ -120,7 +120,41 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key))) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL hr) '4')) (and (= (TOK_TABLE_OR_COL key) '484') (= (TOK_TABLE_OR_COL value) 'val_484')))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + hr + '4' + and + = + TOK_TABLE_OR_COL + key + '484' + = + TOK_TABLE_OR_COL + value + 'val_484' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -129,34 +163,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((key = '484') and (value = 'val_484')) - type: boolean - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((key = '484') and (value = 'val_484')) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: _col0 - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -250,7 +275,44 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL hr) '4')) (and (= (TOK_TABLE_OR_COL key) '238') (= (TOK_TABLE_OR_COL value) 'val_238')))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + hr + '4' + and + = + TOK_TABLE_OR_COL + key + '238' + = + TOK_TABLE_OR_COL + value + 'val_238' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -259,36 +321,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((key = '238') and (value = 'val_238')) - type: boolean - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((key = '238') and (value = 'val_238')) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -383,7 +434,36 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key))) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL hr) '4')) (= (TOK_TABLE_OR_COL value) "3"))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + hr + '4' + = + TOK_TABLE_OR_COL + value + "3" + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -392,34 +472,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (value = '3') - type: boolean - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + predicate: (value = '3') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: _col0 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -512,7 +583,39 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL hr) '4')) (= (TOK_TABLE_OR_COL key) '495'))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + hr + '4' + = + TOK_TABLE_OR_COL + key + '495' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -521,36 +624,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (key = '495') - type: boolean - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + predicate: (key = '495') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out index ff210fa..4aa8903 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out @@ -118,7 +118,46 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL hr) '4')) (= (TOK_TABLE_OR_COL value) 'val_484'))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + hr + '4' + = + TOK_TABLE_OR_COL + value + 'val_484' + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -127,44 +166,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (value = 'val_484') - type: boolean - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + predicate: (value = 'val_484') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + value expressions: _col0 (type: string), _col1 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -258,15 +278,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -313,7 +331,40 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key))) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL hr) '4')) (= (TOK_TABLE_OR_COL key) '406'))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + hr + '4' + = + TOK_TABLE_OR_COL + key + '406' + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -322,38 +373,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (key = '406') - type: boolean - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + predicate: (key = '406') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: _col0 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string + value expressions: _col0 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -404,15 +442,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -462,7 +498,61 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL hr) '4')) (or (and (= (TOK_TABLE_OR_COL key) '484') (= (TOK_TABLE_OR_COL value) 'val_484')) (and (= (TOK_TABLE_OR_COL key) '238') (= (TOK_TABLE_OR_COL value) 'val_238'))))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + hr + '4' + or + and + = + TOK_TABLE_OR_COL + key + '484' + = + TOK_TABLE_OR_COL + value + 'val_484' + and + = + TOK_TABLE_OR_COL + key + '238' + = + TOK_TABLE_OR_COL + value + 'val_238' + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + key + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + value + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -471,44 +561,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (((key = '484') and (value = 'val_484')) or ((key = '238') and (value = 'val_238'))) - type: boolean - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + predicate: (((key = '484') and (value = 'val_484')) or ((key = '238') and (value = 'val_238'))) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string + expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: string - expr: _col1 - type: string + value expressions: _col0 (type: string), _col1 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -602,15 +673,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out index 6565e53..be8acbb 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out @@ -262,7 +262,35 @@ POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=2).value SIMPLE [(src)src.FieldS POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=3).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL hr) '1')) (= (TOK_TABLE_OR_COL key) '145'))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + hr + '1' + = + TOK_TABLE_OR_COL + key + '145' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -271,40 +299,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (key = '145') - type: boolean - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + predicate: (key = '145') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: ds - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 250 dataSize: 2656 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -402,7 +415,31 @@ POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=2).value SIMPLE [(src)src.FieldS POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=3).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTIONSTAR count))) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL hr) '1'))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_FUNCTIONSTAR + count + TOK_WHERE + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + hr + '1' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -411,32 +448,23 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Select Operator - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: - expr: count() - bucketGroup: false + aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: - numRows: 1 dataSize: 8 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: - numRows: 1 dataSize: 8 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: - expr: _col0 - type: bigint + value expressions: _col0 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -487,27 +515,20 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false + aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: - numRows: 1 dataSize: 8 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: - expr: _col0 - type: bigint + expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: - numRows: 1 dataSize: 8 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 1 dataSize: 8 basicStatsState: COMPLETE colStatsState: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -560,7 +581,40 @@ POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=2).value SIMPLE [(src)src.FieldS POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=3).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL hr) '2')) (and (= (TOK_TABLE_OR_COL key) '484') (= (TOK_TABLE_OR_COL value) 'val_484')))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + hr + '2' + and + = + TOK_TABLE_OR_COL + key + '484' + = + TOK_TABLE_OR_COL + value + 'val_484' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -569,40 +623,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((key = '484') and (value = 'val_484')) - type: boolean - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((key = '484') and (value = 'val_484')) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: ds - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -702,7 +741,40 @@ POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=2).value SIMPLE [(src)src.FieldS POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=3).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_daily PARTITION(ds=1,hr=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL hr) '3')) (and (= (TOK_TABLE_OR_COL key) '327') (= (TOK_TABLE_OR_COL value) 'val_327')))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + and + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + hr + '3' + and + = + TOK_TABLE_OR_COL + key + '327' + = + TOK_TABLE_OR_COL + value + 'val_327' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -711,40 +783,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 500 dataSize: 5312 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: ((key = '327') and (value = 'val_327')) - type: boolean - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + predicate: ((key = '327') and (value = 'val_327')) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: ds - type: string - expr: hr - type: string + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 125 dataSize: 1328 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out index b88fc8f..224fd84 100644 --- ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out @@ -181,7 +181,31 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=1).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL x))) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL x) 484))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + x + TOK_WHERE + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + x + 484 + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -190,34 +214,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 2 dataSize: 8 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (x = 484) - type: boolean - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + predicate: (x = 484) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: x - type: int + expressions: x (type: int) outputColumnNames: _col0 - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -311,7 +326,31 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=1).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL x))) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL x) 495))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + x + TOK_WHERE + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + x + 495 + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -320,34 +359,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 2 dataSize: 8 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (x = 495) - type: boolean - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + predicate: (x = 495) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: x - type: int + expressions: x (type: int) outputColumnNames: _col0 - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -437,7 +467,31 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=1).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL x))) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL x) 1))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + x + TOK_WHERE + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + x + 1 + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -446,34 +500,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 2 dataSize: 8 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (x = 1) - type: boolean - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + predicate: (x = 1) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: x - type: int + expressions: x (type: int) outputColumnNames: _col0 - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out index 9e5b01d..3932870 100644 --- ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out @@ -201,7 +201,43 @@ POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=1).y SIMPLE [(src)src.FieldSchema(n POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).y SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL x))) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '1')))) subq)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL x))) (TOK_WHERE (= (TOK_TABLE_OR_COL x) 484)))) + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + x + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '1' + subq + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + x + TOK_WHERE + = + TOK_TABLE_OR_COL + x + 484 + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -210,34 +246,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - subq:fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 6 dataSize: 24 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (x = 484) - type: boolean - Statistics: - numRows: 3 dataSize: 12 basicStatsState: COMPLETE colStatsState: NONE + predicate: (x = 484) (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: x - type: int + expressions: x (type: int) outputColumnNames: _col0 - Statistics: - numRows: 3 dataSize: 12 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 3 dataSize: 12 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -335,7 +362,58 @@ POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=1).y SIMPLE [(src)src.FieldSchema(n POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).y SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL x) x1) (TOK_SELEXPR (TOK_TABLE_OR_COL y) y1)) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '1')))) subq)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL x1)) (TOK_SELEXPR (TOK_TABLE_OR_COL y1))) (TOK_WHERE (= (TOK_TABLE_OR_COL x1) 484)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL x1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL y1))))) + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + x + x1 + TOK_SELEXPR + TOK_TABLE_OR_COL + y + y1 + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '1' + subq + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + x1 + TOK_SELEXPR + TOK_TABLE_OR_COL + y1 + TOK_WHERE + = + TOK_TABLE_OR_COL + x1 + 484 + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + x1 + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + y1 + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -344,44 +422,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - subq:fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 0 dataSize: 24 basicStatsState: PARTIAL colStatsState: NONE + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (x = 484) - type: boolean - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + predicate: (x = 484) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: x - type: int - expr: y - type: string + expressions: x (type: int), y (type: string) outputColumnNames: _col0, _col1 - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: int - expr: _col1 - type: string + key expressions: _col0 (type: int), _col1 (type: string) sort order: ++ - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: int - expr: _col1 - type: string + value expressions: _col0 (type: int), _col1 (type: string) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -432,15 +491,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -491,7 +548,38 @@ POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=1).y SIMPLE [(src)src.FieldSchema(n POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).y SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL y)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1') (= (TOK_TABLE_OR_COL x) 484))) (TOK_GROUPBY (TOK_TABLE_OR_COL y)))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + y + TOK_SELEXPR + TOK_FUNCTION + count + 1 + TOK_WHERE + and + = + TOK_TABLE_OR_COL + ds + '1' + = + TOK_TABLE_OR_COL + x + 484 + TOK_GROUPBY + TOK_TABLE_OR_COL + y + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -500,52 +588,32 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 0 dataSize: 24 basicStatsState: PARTIAL colStatsState: NONE + Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (x = 484) - type: boolean - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + predicate: (x = 484) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: y - type: string + expressions: y (type: string) outputColumnNames: y - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: y - type: string + aggregations: count(1) + keys: y (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE tag: -1 - value expressions: - expr: _col1 - type: bigint + value expressions: _col1 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -596,32 +664,21 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 0 dataSize: 0 basicStatsState: NONE colStatsState: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -672,7 +729,54 @@ POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=1).y SIMPLE [(src)src.FieldSchema(n POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).y SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL x)) (TOK_SELEXPR (TOK_FUNCTION count 1) c)) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '1')) (TOK_GROUPBY (TOK_TABLE_OR_COL x)))) subq)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL x)) (TOK_SELEXPR (TOK_TABLE_OR_COL c))) (TOK_WHERE (= (TOK_TABLE_OR_COL x) 484)))) + +TOK_QUERY + TOK_FROM + TOK_SUBQUERY + TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + x + TOK_SELEXPR + TOK_FUNCTION + count + 1 + c + TOK_WHERE + = + TOK_TABLE_OR_COL + ds + '1' + TOK_GROUPBY + TOK_TABLE_OR_COL + x + subq + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + x + TOK_SELEXPR + TOK_TABLE_OR_COL + c + TOK_WHERE + = + TOK_TABLE_OR_COL + x + 484 + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -681,52 +785,32 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - subq:fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 6 dataSize: 24 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (x = 484) - type: boolean - Statistics: - numRows: 3 dataSize: 12 basicStatsState: COMPLETE colStatsState: NONE + predicate: (x = 484) (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: x - type: int + expressions: x (type: int) outputColumnNames: x - Statistics: - numRows: 3 dataSize: 12 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: x - type: int + aggregations: count(1) + keys: x (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: - numRows: 3 dataSize: 12 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: int + key expressions: _col0 (type: int) sort order: + - Map-reduce partition columns: - expr: _col0 - type: int - Statistics: - numRows: 3 dataSize: 12 basicStatsState: COMPLETE colStatsState: NONE + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col1 - type: bigint + value expressions: _col1 (type: bigint) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -777,32 +861,21 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: int + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: int - expr: _col1 - type: bigint + expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 1 dataSize: 4 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out index 9dfdb28..27f31a8 100644 --- ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out @@ -265,7 +265,36 @@ POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=3).x EXPRESSION [(src)src.FieldSche POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=3).y SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=3).z SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME fact_daily))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL x))) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1') (not (= (TOK_TABLE_OR_COL x) 86)))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL x))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + fact_daily + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + x + TOK_WHERE + and + = + TOK_TABLE_OR_COL + ds + '1' + not + = + TOK_TABLE_OR_COL + x + 86 + TOK_ORDERBY + TOK_TABSORTCOLNAMEASC + TOK_TABLE_OR_COL + x + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -274,38 +303,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - fact_daily + Map Operator Tree: TableScan alias: fact_daily - Statistics: - numRows: 29 dataSize: 117 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 29 Data size: 117 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (not (x = 86)) - type: boolean - Statistics: - numRows: 15 dataSize: 60 basicStatsState: COMPLETE colStatsState: NONE + predicate: (not (x = 86)) (type: boolean) + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: x - type: int + expressions: x (type: int) outputColumnNames: _col0 - Statistics: - numRows: 15 dataSize: 60 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: int + key expressions: _col0 (type: int) sort order: + - Statistics: - numRows: 15 dataSize: 60 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: - expr: _col0 - type: int + value expressions: _col0 (type: int) Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -442,15 +458,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Extract - Statistics: - numRows: 15 dataSize: 60 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: - numRows: 15 dataSize: 60 basicStatsState: COMPLETE colStatsState: NONE + Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat diff --git ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out index 12bb869..2a267c7 100644 --- ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out +++ ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out @@ -50,9 +50,6 @@ POSTHOOK: query: -- a simple join query with skew on both the tables on the join EXPLAIN SELECT * FROM T1 a JOIN T2 b ON a.key = b.key POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME T1) a) (TOK_TABREF (TOK_TABNAME T2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-3 is a root stage @@ -61,49 +58,31 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - subquery1:a + Map Operator Tree: TableScan alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (not ((key = '2') or (key = '3'))) - type: boolean + predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: val - type: string - subquery1:b + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (not ((key = '2') or (key = '3'))) - type: boolean + predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: val - type: string + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -111,22 +90,15 @@ STAGE PLANS: condition expressions: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} - handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -134,49 +106,31 @@ STAGE PLANS: Stage: Stage-3 Map Reduce - Alias -> Map Operator Tree: - a + Map Operator Tree: TableScan - alias: a + alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: ((key = '2') or (key = '3')) - type: boolean + predicate: ((key = '2') or (key = '3')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: val - type: string - b + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan - alias: b + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: ((key = '2') or (key = '3')) - type: boolean + predicate: ((key = '2') or (key = '3')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: val - type: string + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -184,22 +138,15 @@ STAGE PLANS: condition expressions: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} - handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -209,7 +156,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: SELECT * FROM T1 a JOIN T2 b ON a.key = b.key ORDER BY a.key, b.key, a.val, b.val PREHOOK: type: QUERY @@ -238,9 +184,6 @@ POSTHOOK: query: -- test outer joins also EXPLAIN SELECT a.*, b.* FROM T1 a RIGHT OUTER JOIN T2 b ON a.key = b.key POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_TABREF (TOK_TABNAME T1) a) (TOK_TABREF (TOK_TABNAME T2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME a))) (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME b)))))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-3 is a root stage @@ -249,49 +192,31 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - subquery1:a + Map Operator Tree: TableScan alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (not ((key = '2') or (key = '3'))) - type: boolean + predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: val - type: string - subquery1:b + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (not ((key = '2') or (key = '3'))) - type: boolean + predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: val - type: string + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -299,22 +224,15 @@ STAGE PLANS: condition expressions: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} - handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -322,49 +240,31 @@ STAGE PLANS: Stage: Stage-3 Map Reduce - Alias -> Map Operator Tree: - a + Map Operator Tree: TableScan - alias: a + alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: ((key = '2') or (key = '3')) - type: boolean + predicate: ((key = '2') or (key = '3')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: val - type: string - b + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan - alias: b + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: ((key = '2') or (key = '3')) - type: boolean + predicate: ((key = '2') or (key = '3')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: val - type: string + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -372,22 +272,15 @@ STAGE PLANS: condition expressions: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} - handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -397,7 +290,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: SELECT a.*, b.* FROM T1 a RIGHT OUTER JOIN T2 b ON a.key = b.key ORDER BY a.key, b.key, a.val, b.val PREHOOK: type: QUERY @@ -431,9 +323,6 @@ POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE DEST1 SELECT * FROM T1 a JOIN T2 b ON a.key = b.key POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME T1) a) (TOK_TABREF (TOK_TABNAME T2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME DEST1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-3 @@ -442,49 +331,31 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - subquery1:a + Map Operator Tree: TableScan alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (not ((key = '2') or (key = '3'))) - type: boolean + predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: val - type: string - subquery1:b + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (not ((key = '2') or (key = '3'))) - type: boolean + predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: val - type: string + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -492,22 +363,15 @@ STAGE PLANS: condition expressions: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} - handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -526,49 +390,31 @@ STAGE PLANS: Stage: Stage-3 Map Reduce - Alias -> Map Operator Tree: - a + Map Operator Tree: TableScan - alias: a + alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: ((key = '2') or (key = '3')) - type: boolean + predicate: ((key = '2') or (key = '3')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: val - type: string - b + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan - alias: b + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: ((key = '2') or (key = '3')) - type: boolean + predicate: ((key = '2') or (key = '3')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: val - type: string + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -576,29 +422,21 @@ STAGE PLANS: condition expressions: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} - handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - PREHOOK: query: INSERT OVERWRITE TABLE DEST1 SELECT * FROM T1 a JOIN T2 b ON a.key = b.key PREHOOK: type: QUERY @@ -647,9 +485,6 @@ POSTHOOK: Lineage: dest1.key1 SIMPLE [(t1)a.FieldSchema(name:key, type:string, c POSTHOOK: Lineage: dest1.key2 SIMPLE [(t2)b.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: dest1.val1 SIMPLE [(t1)a.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: dest1.val2 SIMPLE [(t2)b.FieldSchema(name:val, type:string, comment:null), ] -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_TABREF (TOK_TABNAME T1) a) (TOK_TABREF (TOK_TABNAME T2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME DEST1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-3 @@ -658,49 +493,31 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - subquery1:a + Map Operator Tree: TableScan alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (not ((key = '2') or (key = '3'))) - type: boolean + predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: val - type: string - subquery1:b + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (not ((key = '2') or (key = '3'))) - type: boolean + predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: val - type: string + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -708,22 +525,15 @@ STAGE PLANS: condition expressions: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} - handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -742,49 +552,31 @@ STAGE PLANS: Stage: Stage-3 Map Reduce - Alias -> Map Operator Tree: - a + Map Operator Tree: TableScan - alias: a + alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: ((key = '2') or (key = '3')) - type: boolean + predicate: ((key = '2') or (key = '3')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: val - type: string - b + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan - alias: b + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: ((key = '2') or (key = '3')) - type: boolean + predicate: ((key = '2') or (key = '3')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: val - type: string + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -792,29 +584,21 @@ STAGE PLANS: condition expressions: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} - handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - PREHOOK: query: INSERT OVERWRITE TABLE DEST1 SELECT * FROM T1 a RIGHT OUTER JOIN T2 b ON a.key = b.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out index 9800c54..5c92675 100644 --- ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out +++ ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out @@ -57,9 +57,6 @@ POSTHOOK: query: -- This is to test the union->selectstar->filesink and skewjoin EXPLAIN SELECT a.*, b.*, c.* FROM T1 a JOIN T2 b ON a.key = b.key JOIN T3 c on a.key = c.key POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME T1) a) (TOK_TABREF (TOK_TABNAME T2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key))) (TOK_TABREF (TOK_TABNAME T3) c) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL c) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME a))) (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME b))) (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME c)))))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-4 is a root stage @@ -68,70 +65,43 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - subquery1:a + Map Operator Tree: TableScan - alias: a + alias: c + Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (not (((key = '2') or (key = '8')) or (key = '3'))) - type: boolean + predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: val - type: string - subquery1:b + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan - alias: b + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (not (((key = '2') or (key = '8')) or (key = '3'))) - type: boolean + predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: val - type: string - subquery1:c + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan - alias: c + alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (not (((key = '2') or (key = '8')) or (key = '3'))) - type: boolean + predicate: (not (((key = '2') or (key = '8')) or (key = '3'))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 2 - value expressions: - expr: key - type: string - expr: val - type: string + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -141,26 +111,15 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} 2 {VALUE._col0} {VALUE._col1} - handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string - expr: _col8 - type: string - expr: _col9 - type: string + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -168,70 +127,43 @@ STAGE PLANS: Stage: Stage-4 Map Reduce - Alias -> Map Operator Tree: - a + Map Operator Tree: TableScan - alias: a + alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (((key = '2') or (key = '8')) or (key = '3')) - type: boolean + predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - expr: val - type: string - b + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan - alias: b + alias: c + Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (((key = '2') or (key = '8')) or (key = '3')) - type: boolean + predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: key - type: string - expr: val - type: string - c + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) TableScan - alias: c + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (((key = '2') or (key = '8')) or (key = '3')) - type: boolean + predicate: (((key = '2') or (key = '8')) or (key = '3')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 2 - value expressions: - expr: key - type: string - expr: val - type: string + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string), val (type: string) Reduce Operator Tree: Join Operator condition map: @@ -241,26 +173,15 @@ STAGE PLANS: 0 {VALUE._col0} {VALUE._col1} 1 {VALUE._col0} {VALUE._col1} 2 {VALUE._col0} {VALUE._col1} - handleSkewJoin: false outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: string - expr: _col4 - type: string - expr: _col5 - type: string - expr: _col8 - type: string - expr: _col9 - type: string + expressions: _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string), _col8 (type: string), _col9 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -270,7 +191,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: SELECT a.*, b.*, c.* FROM T1 a JOIN T2 b ON a.key = b.key JOIN T3 c on a.key = c.key ORDER BY a.key, b.key, c.key, a.val, b.val, c.val PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats_partscan_1_23.q.out ql/src/test/results/clientpositive/stats_partscan_1_23.q.out index 1799036..beebfad 100644 --- ql/src/test/results/clientpositive/stats_partscan_1_23.q.out +++ ql/src/test/results/clientpositive/stats_partscan_1_23.q.out @@ -109,9 +109,6 @@ POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).k POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -ABSTRACT SYNTAX TREE: - (TOK_ANALYZE (TOK_TAB (TOK_TABNAME analyze_srcpart_partial_scan) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr 11))) partialscan) - STAGE DEPENDENCIES: Stage-2 is a root stage Stage-1 depends on stages: Stage-2 @@ -123,7 +120,6 @@ STAGE PLANS: Stage: Stage-1 Stats-Aggr Operator - PREHOOK: query: analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan PREHOOK: type: QUERY PREHOOK: Input: default@analyze_srcpart_partial_scan diff --git ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out index 73207b0..d345dae 100644 --- ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out +++ ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out @@ -71,7 +71,30 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: test_tab PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: test_tab PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME test_tab))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (AND (= (TOK_TABLE_OR_COL part) '1') (= (TOK_TABLE_OR_COL key) '484'))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + test_tab + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + AND + = + TOK_TABLE_OR_COL + part + '1' + = + TOK_TABLE_OR_COL + key + '484' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -80,30 +103,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - test_tab + Map Operator Tree: TableScan alias: test_tab + Statistics: Num rows: 8 Data size: 1761 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (key = '484') - type: boolean + predicate: (key = '484') (type: boolean) + Statistics: Num rows: 4 Data size: 880 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: part - type: string + expressions: key (type: string), value (type: string), part (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 880 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 + Statistics: Num rows: 4 Data size: 880 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -174,7 +192,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: SELECT * FROM test_tab WHERE part = '1' AND key = '484' PREHOOK: type: QUERY PREHOOK: Input: default@test_tab @@ -195,7 +212,30 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: test_tab PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: test_tab PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME test_tab))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (AND (= (TOK_TABLE_OR_COL part) '1') (= (TOK_TABLE_OR_COL key) '0'))))) + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + test_tab + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_WHERE + AND + = + TOK_TABLE_OR_COL + part + '1' + = + TOK_TABLE_OR_COL + key + '0' + STAGE DEPENDENCIES: Stage-1 is a root stage @@ -204,30 +244,25 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - test_tab + Map Operator Tree: TableScan alias: test_tab + Statistics: Num rows: 8 Data size: 1761 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: - expr: (key = '0') - type: boolean + predicate: (key = '0') (type: boolean) + Statistics: Num rows: 4 Data size: 880 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: part - type: string + expressions: key (type: string), value (type: string), part (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 880 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 + Statistics: Num rows: 4 Data size: 880 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -298,7 +333,6 @@ STAGE PLANS: Fetch Operator limit: -1 - PREHOOK: query: SELECT * FROM test_tab WHERE part = '1' AND key = '0' PREHOOK: type: QUERY PREHOOK: Input: default@test_tab diff --git ql/src/test/results/clientpositive/union_remove_1.q.out ql/src/test/results/clientpositive/union_remove_1.q.out index 785cd03..74d20bc 100644 --- ql/src/test/results/clientpositive/union_remove_1.q.out +++ ql/src/test/results/clientpositive/union_remove_1.q.out @@ -52,9 +52,6 @@ FROM ( SELECT key, count(1) as values from inputTbl1 group by key ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -63,56 +60,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -131,63 +112,46 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_10.q.out ql/src/test/results/clientpositive/union_remove_10.q.out index 25eeb50..76cc5fc 100644 --- ql/src/test/results/clientpositive/union_remove_10.q.out +++ ql/src/test/results/clientpositive/union_remove_10.q.out @@ -68,9 +68,6 @@ select * FROM ( ) a )b POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 values)))) (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 values))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))) b)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -85,56 +82,39 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:b-subquery2-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -142,46 +122,38 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan Union + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - null-subquery2:b-subquery2-subquery2:a-subquery2:inputtbl1 TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: UDFToLong(2) - type: bigint + expressions: key (type: string), UDFToLong(2) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -221,27 +193,23 @@ STAGE PLANS: Stage: Stage-8 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:b-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: UDFToLong(1) - type: bigint + expressions: key (type: string), UDFToLong(1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_11.q.out ql/src/test/results/clientpositive/union_remove_11.q.out index 2e579d4..68efb49 100644 --- ql/src/test/results/clientpositive/union_remove_11.q.out +++ ql/src/test/results/clientpositive/union_remove_11.q.out @@ -68,9 +68,6 @@ select * FROM ( ) a )b POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 values)))) (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 values)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 3 values))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))) b)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 @@ -83,96 +80,77 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:b-subquery1:inputtbl1 - TableScan - alias: inputtbl1 - Select Operator - expressions: - expr: key - type: string - expr: 1 - type: int - outputColumnNames: _col0, _col1 - Union - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 - null-subquery2:b-subquery2-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: 2 - type: int + expressions: key (type: string), 2 (type: int) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: int + expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - null-subquery2:b-subquery2-subquery2:a-subquery2:inputtbl1 TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: 3 - type: int + expressions: key (type: string), 1 (type: int) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: int + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 + TableScan + alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string), 3 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + Union + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -210,7 +188,6 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_12.q.out ql/src/test/results/clientpositive/union_remove_12.q.out index 5794ae1..d2dd47c 100644 --- ql/src/test/results/clientpositive/union_remove_12.q.out +++ ql/src/test/results/clientpositive/union_remove_12.q.out @@ -60,9 +60,6 @@ select a.key as key, b.val as values FROM inputTbl1 a join inputTbl1 b on a.key=b.key )c POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 values)))) (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME inputTbl1) a) (TOK_TABREF (TOK_TABNAME inputTbl1) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) val) values))))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-8 is a root stage Stage-6 depends on stages: Stage-8, Stage-9 , consists of Stage-3, Stage-2, Stage-4 @@ -71,26 +68,22 @@ STAGE DEPENDENCIES: Stage-2 Stage-4 Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-9 is a root stage STAGE PLANS: Stage: Stage-8 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:c-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: UDFToLong(UDFToString(1)) - type: bigint + expressions: key (type: string), UDFToLong(UDFToString(1)) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -128,54 +121,30 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-10 - Map Reduce Local Work - Alias -> Map Local Tables: - null-subquery2:c-subquery2:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - null-subquery2:c-subquery2:a - TableScan - alias: a - HashTable Sink Operator - condition expressions: - 0 {key} - 1 {val} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - Position of Big Table: 1 - Stage: Stage-9 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:c-subquery2:b + Map Operator Tree: TableScan alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: 0 {key} 1 {val} - handleSkewJoin: false keys: - 0 [Column[key]] - 1 [Column[key]] + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col5 - Position of Big Table: 1 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col5) - type: bigint + expressions: _col0 (type: string), UDFToLong(_col5) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -183,7 +152,15 @@ STAGE PLANS: name: default.outputtbl1 Local Work: Map Reduce Local Work - + Alias -> Map Local Tables: + null-subquery2:c-subquery2:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery2:c-subquery2:a + TableScan + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM diff --git ql/src/test/results/clientpositive/union_remove_13.q.out ql/src/test/results/clientpositive/union_remove_13.q.out index 581472a..fa0849c 100644 --- ql/src/test/results/clientpositive/union_remove_13.q.out +++ ql/src/test/results/clientpositive/union_remove_13.q.out @@ -60,9 +60,6 @@ select a.key as key, b.val as values FROM inputTbl1 a join inputTbl1 b on a.key=b.key )c POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME inputTbl1) a) (TOK_TABREF (TOK_TABNAME inputTbl1) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) val) values))))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-8 is a root stage Stage-6 depends on stages: Stage-8, Stage-9 , consists of Stage-3, Stage-2, Stage-4 @@ -71,62 +68,45 @@ STAGE DEPENDENCIES: Stage-2 Stage-4 Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-9 is a root stage STAGE PLANS: Stage: Stage-8 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:c-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(UDFToString(_col1)) - type: bigint + expressions: _col0 (type: string), UDFToLong(UDFToString(_col1)) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -164,54 +144,30 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-10 - Map Reduce Local Work - Alias -> Map Local Tables: - null-subquery2:c-subquery2:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - null-subquery2:c-subquery2:a - TableScan - alias: a - HashTable Sink Operator - condition expressions: - 0 {key} - 1 {val} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - Position of Big Table: 1 - Stage: Stage-9 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:c-subquery2:b + Map Operator Tree: TableScan alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: 0 {key} 1 {val} - handleSkewJoin: false keys: - 0 [Column[key]] - 1 [Column[key]] + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col5 - Position of Big Table: 1 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col5) - type: bigint + expressions: _col0 (type: string), UDFToLong(_col5) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -219,7 +175,15 @@ STAGE PLANS: name: default.outputtbl1 Local Work: Map Reduce Local Work - + Alias -> Map Local Tables: + null-subquery2:c-subquery2:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery2:c-subquery2:a + TableScan + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM diff --git ql/src/test/results/clientpositive/union_remove_14.q.out ql/src/test/results/clientpositive/union_remove_14.q.out index 3537d7b..f837dea 100644 --- ql/src/test/results/clientpositive/union_remove_14.q.out +++ ql/src/test/results/clientpositive/union_remove_14.q.out @@ -62,9 +62,6 @@ select a.key as key, b.val as values FROM inputTbl1 a join inputTbl1 b on a.key=b.key )c POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 values)))) (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME inputTbl1) a) (TOK_TABREF (TOK_TABNAME inputTbl1) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) val) values))))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-8 is a root stage Stage-6 depends on stages: Stage-8, Stage-9 , consists of Stage-3, Stage-2, Stage-4 @@ -73,26 +70,22 @@ STAGE DEPENDENCIES: Stage-2 Stage-4 Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-9 is a root stage STAGE PLANS: Stage: Stage-8 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:c-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: UDFToLong(UDFToString(1)) - type: bigint + expressions: key (type: string), UDFToLong(UDFToString(1)) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -130,54 +123,30 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-10 - Map Reduce Local Work - Alias -> Map Local Tables: - null-subquery2:c-subquery2:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - null-subquery2:c-subquery2:a - TableScan - alias: a - HashTable Sink Operator - condition expressions: - 0 {key} - 1 {val} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - Position of Big Table: 1 - Stage: Stage-9 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:c-subquery2:b + Map Operator Tree: TableScan alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: 0 {key} 1 {val} - handleSkewJoin: false keys: - 0 [Column[key]] - 1 [Column[key]] + 0 key (type: string) + 1 key (type: string) outputColumnNames: _col0, _col5 - Position of Big Table: 1 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col5) - type: bigint + expressions: _col0 (type: string), UDFToLong(_col5) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -185,7 +154,15 @@ STAGE PLANS: name: default.outputtbl1 Local Work: Map Reduce Local Work - + Alias -> Map Local Tables: + null-subquery2:c-subquery2:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery2:c-subquery2:a + TableScan + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM diff --git ql/src/test/results/clientpositive/union_remove_15.q.out ql/src/test/results/clientpositive/union_remove_15.q.out index fbd76f8..d439ec7 100644 --- ql/src/test/results/clientpositive/union_remove_15.q.out +++ ql/src/test/results/clientpositive/union_remove_15.q.out @@ -58,9 +58,6 @@ FROM ( SELECT key, count(1) as values, '2' as ds from inputTbl1 group by key ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values) (TOK_SELEXPR '1' ds)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values) (TOK_SELEXPR '2' ds)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1) (TOK_PARTSPEC (TOK_PARTVAL ds)))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -69,58 +66,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - expr: '2' - type: string + expressions: _col0 (type: string), _col1 (type: bigint), '2' (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -141,65 +120,46 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - expr: '1' - type: string + expressions: _col0 (type: string), _col1 (type: bigint), '1' (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 partition (ds) SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_16.q.out ql/src/test/results/clientpositive/union_remove_16.q.out index 18edba7..eeba6a3 100644 --- ql/src/test/results/clientpositive/union_remove_16.q.out +++ ql/src/test/results/clientpositive/union_remove_16.q.out @@ -56,9 +56,6 @@ FROM ( SELECT key, count(1) as values, '2' as ds from inputTbl1 group by key ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values) (TOK_SELEXPR '1' ds)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values) (TOK_SELEXPR '2' ds)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1) (TOK_PARTSPEC (TOK_PARTVAL ds)))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 @@ -72,58 +69,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - expr: '2' - type: string + expressions: _col0 (type: string), _col1 (type: bigint), '2' (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -165,65 +144,46 @@ STAGE PLANS: Stage: Stage-7 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - expr: '1' - type: string + expressions: _col0 (type: string), _col1 (type: bigint), '1' (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 partition (ds) SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_17.q.out ql/src/test/results/clientpositive/union_remove_17.q.out index 73531d3..1fdc4e8 100644 --- ql/src/test/results/clientpositive/union_remove_17.q.out +++ ql/src/test/results/clientpositive/union_remove_17.q.out @@ -52,9 +52,6 @@ FROM ( SELECT key, 2 as values, '2' as ds from inputTbl1 ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 values) (TOK_SELEXPR '1' ds)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 values) (TOK_SELEXPR '2' ds))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1) (TOK_PARTSPEC (TOK_PARTVAL ds)))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -62,62 +59,44 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: 1 - type: int - expr: '1' - type: string + expressions: key (type: string), 2 (type: int), '2' (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - expr: _col2 - type: string + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - null-subquery2:a-subquery2:inputtbl1 TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: 2 - type: int - expr: '2' - type: string + expressions: key (type: string), 1 (type: int), '1' (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - expr: _col2 - type: string + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -136,7 +115,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 partition (ds) SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_18.q.out ql/src/test/results/clientpositive/union_remove_18.q.out index bc8046f..04c5e74 100644 --- ql/src/test/results/clientpositive/union_remove_18.q.out +++ ql/src/test/results/clientpositive/union_remove_18.q.out @@ -56,9 +56,6 @@ FROM ( SELECT key, count(1) as values, ds from inputTbl1 group by key, ds ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values) (TOK_SELEXPR (TOK_TABLE_OR_COL ds))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL ds)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values) (TOK_SELEXPR (TOK_TABLE_OR_COL ds))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL ds))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1) (TOK_PARTSPEC (TOK_PARTVAL ds)))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -67,68 +64,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: ds - type: string + expressions: key (type: string), ds (type: string) outputColumnNames: key, ds + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string - expr: ds - type: string + aggregations: count(1) + keys: key (type: string), ds (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: - expr: _col0 - type: string - expr: _col1 - type: string - tag: -1 - value expressions: - expr: _col2 - type: bigint + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string - expr: KEY._col1 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col2 - type: bigint - expr: _col1 - type: string + expressions: _col0 (type: string), _col2 (type: bigint), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -149,75 +118,46 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: ds - type: string + expressions: key (type: string), ds (type: string) outputColumnNames: key, ds + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string - expr: ds - type: string + aggregations: count(1) + keys: key (type: string), ds (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string - expr: _col1 - type: string + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: - expr: _col0 - type: string - expr: _col1 - type: string - tag: -1 - value expressions: - expr: _col2 - type: bigint + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string - expr: KEY._col1 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col2 - type: bigint - expr: _col1 - type: string + expressions: _col0 (type: string), _col2 (type: bigint), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 partition (ds) SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_19.q.out ql/src/test/results/clientpositive/union_remove_19.q.out index 44eb72b..d442585 100644 --- ql/src/test/results/clientpositive/union_remove_19.q.out +++ ql/src/test/results/clientpositive/union_remove_19.q.out @@ -52,9 +52,6 @@ FROM ( SELECT key, count(1) as values from inputTbl1 group by key ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) values))))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -63,56 +60,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -131,63 +112,46 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT a.key, a.values FROM ( @@ -283,9 +247,6 @@ FROM ( POSTHOOK: type: QUERY POSTHOOK: Lineage: outputtbl1.key EXPRESSION [(inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), (inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl1.values EXPRESSION [(inputtbl1)inputtbl1.null, (inputtbl1)inputtbl1.null, ] -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) values))) (TOK_WHERE (= (. (TOK_TABLE_OR_COL a) key) 7)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -294,60 +255,43 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (key = 7) - type: boolean + predicate: (key = 7) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -366,67 +310,49 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: - expr: (key = 7) - type: boolean + predicate: (key = 7) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT a.key, a.values FROM ( @@ -495,9 +421,6 @@ POSTHOOK: Lineage: outputtbl1.key EXPRESSION [(inputtbl1)inputtbl1.FieldSchema(n POSTHOOK: Lineage: outputtbl1.key EXPRESSION [(inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), (inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl1.values EXPRESSION [(inputtbl1)inputtbl1.null, (inputtbl1)inputtbl1.null, ] POSTHOOK: Lineage: outputtbl1.values EXPRESSION [(inputtbl1)inputtbl1.null, (inputtbl1)inputtbl1.null, ] -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL a) key)) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) values))))) b)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL values))) (TOK_WHERE (>= (. (TOK_TABLE_OR_COL b) key) 7)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -506,67 +429,47 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - b-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: (_col0 + _col0) - type: double - expr: _col1 - type: bigint + expressions: (_col0 + _col0) (type: double), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: - expr: (_col0 >= 7.0) - type: boolean + predicate: (_col0 >= 7.0) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: double - expr: _col1 - type: bigint + expressions: _col0 (type: double), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -585,74 +488,53 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - b-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: (_col0 + _col0) - type: double - expr: _col1 - type: bigint + expressions: (_col0 + _col0) (type: double), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator - predicate: - expr: (_col0 >= 7.0) - type: boolean + predicate: (_col0 >= 7.0) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: double - expr: _col1 - type: bigint + expressions: _col0 (type: double), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 select key, values from ( diff --git ql/src/test/results/clientpositive/union_remove_2.q.out ql/src/test/results/clientpositive/union_remove_2.q.out index 9157983..b922366 100644 --- ql/src/test/results/clientpositive/union_remove_2.q.out +++ ql/src/test/results/clientpositive/union_remove_2.q.out @@ -58,9 +58,6 @@ FROM ( SELECT key, 2 as values from inputTbl1 ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 values))))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 values))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2, Stage-3 @@ -70,20 +67,17 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: UDFToLong(2) - type: bigint + expressions: key (type: string), UDFToLong(2) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -102,56 +96,40 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1-subquery1:a-subquery1-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -160,27 +138,23 @@ STAGE PLANS: Stage: Stage-3 Map Reduce - Alias -> Map Operator Tree: - null-subquery1-subquery2:a-subquery1-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: UDFToLong(1) - type: bigint + expressions: key (type: string), UDFToLong(1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_20.q.out ql/src/test/results/clientpositive/union_remove_20.q.out index decdbc4..9530c90 100644 --- ql/src/test/results/clientpositive/union_remove_20.q.out +++ ql/src/test/results/clientpositive/union_remove_20.q.out @@ -54,9 +54,6 @@ FROM ( SELECT key, count(1) as values from inputTbl1 group by key ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) values)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key))))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -65,56 +62,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col1 - type: bigint - expr: _col0 - type: string + expressions: _col1 (type: bigint), _col0 (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -133,63 +114,46 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col1 - type: bigint - expr: _col0 - type: string + expressions: _col1 (type: bigint), _col0 (type: string) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT a.values, a.key FROM ( diff --git ql/src/test/results/clientpositive/union_remove_21.q.out ql/src/test/results/clientpositive/union_remove_21.q.out index 4446085..f737744 100644 --- ql/src/test/results/clientpositive/union_remove_21.q.out +++ ql/src/test/results/clientpositive/union_remove_21.q.out @@ -54,9 +54,6 @@ FROM ( SELECT key, count(1) as values from inputTbl1 group by key ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key))))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -65,54 +62,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string + expressions: _col0 (type: string) outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -131,61 +114,46 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string + expressions: _col0 (type: string) outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT a.key FROM ( diff --git ql/src/test/results/clientpositive/union_remove_22.q.out ql/src/test/results/clientpositive/union_remove_22.q.out index 8faf0ba..f095e5b 100644 --- ql/src/test/results/clientpositive/union_remove_22.q.out +++ ql/src/test/results/clientpositive/union_remove_22.q.out @@ -52,9 +52,6 @@ FROM ( SELECT key, count(1) as values from inputTbl1 group by key ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) values)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) values))))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -63,58 +60,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -133,65 +112,46 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT a.key, a.values, a.values FROM ( @@ -290,9 +250,6 @@ POSTHOOK: type: QUERY POSTHOOK: Lineage: outputtbl1.key EXPRESSION [(inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), (inputtbl1)inputtbl1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl1.values EXPRESSION [(inputtbl1)inputtbl1.null, (inputtbl1)inputtbl1.null, ] POSTHOOK: Lineage: outputtbl1.values2 EXPRESSION [(inputtbl1)inputtbl1.null, (inputtbl1)inputtbl1.null, ] -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (TOK_FUNCTION concat (. (TOK_TABLE_OR_COL a) values) (. (TOK_TABLE_OR_COL a) values))) (TOK_SELEXPR (TOK_FUNCTION concat (. (TOK_TABLE_OR_COL a) values) (. (TOK_TABLE_OR_COL a) values)))))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -301,58 +258,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(concat(_col1, _col1)) - type: bigint - expr: UDFToLong(concat(_col1, _col1)) - type: bigint + expressions: _col0 (type: string), UDFToLong(concat(_col1, _col1)) (type: bigint), UDFToLong(concat(_col1, _col1)) (type: bigint) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -371,65 +310,46 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(concat(_col1, _col1)) - type: bigint - expr: UDFToLong(concat(_col1, _col1)) - type: bigint + expressions: _col0 (type: string), UDFToLong(concat(_col1, _col1)) (type: bigint), UDFToLong(concat(_col1, _col1)) (type: bigint) outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT a.key, concat(a.values, a.values), concat(a.values, a.values) FROM ( diff --git ql/src/test/results/clientpositive/union_remove_23.q.out ql/src/test/results/clientpositive/union_remove_23.q.out index 4ebbe22..51b8e09 100644 --- ql/src/test/results/clientpositive/union_remove_23.q.out +++ ql/src/test/results/clientpositive/union_remove_23.q.out @@ -56,9 +56,6 @@ FROM ( SELECT key, count(1) as values from inputTbl1 group by key ) subq2 POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME inputTbl1) a) (TOK_TABREF (TOK_TABNAME inputTbl1) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) val))))) subq)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) subq2)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-3 @@ -68,56 +65,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:subq2-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -136,34 +117,24 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:subq2-subquery1:subq:a + Map Operator Tree: TableScan - alias: a + alias: b + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - null-subquery1:subq2-subquery1:subq:b + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE TableScan - alias: b + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: key - type: string + key expressions: key (type: string) sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: key (type: string) Reduce Operator Tree: Join Operator condition map: @@ -171,25 +142,20 @@ STAGE PLANS: condition expressions: 0 {VALUE._col0} 1 - handleSkewJoin: false outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string + expressions: _col0 (type: string) outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: _col0 - type: string + aggregations: count(1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -197,48 +163,34 @@ STAGE PLANS: Stage: Stage-3 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 33 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_24.q.out ql/src/test/results/clientpositive/union_remove_24.q.out index 6bc20d8..a54a478 100644 --- ql/src/test/results/clientpositive/union_remove_24.q.out +++ ql/src/test/results/clientpositive/union_remove_24.q.out @@ -50,9 +50,6 @@ SELECT * FROM SELECT CAST(key AS BIGINT) AS key, count(1) as values FROM inputTbl1 group by key ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION TOK_DOUBLE (TOK_TABLE_OR_COL key)) key) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION TOK_BIGINT (TOK_TABLE_OR_COL key)) key) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -61,56 +58,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: UDFToDouble(UDFToLong(_col0)) - type: double - expr: _col1 - type: bigint + expressions: UDFToDouble(UDFToLong(_col0)) (type: double), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -129,63 +110,46 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: UDFToDouble(_col0) - type: double - expr: _col1 - type: bigint + expressions: UDFToDouble(_col0) (type: double), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_3.q.out ql/src/test/results/clientpositive/union_remove_3.q.out index f475aba..f7a64cd 100644 --- ql/src/test/results/clientpositive/union_remove_3.q.out +++ ql/src/test/results/clientpositive/union_remove_3.q.out @@ -58,9 +58,6 @@ FROM ( SELECT key, 3 as values from inputTbl1 ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 values)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 values))))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 3 values))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -68,80 +65,65 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery1-subquery1:a-subquery1-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: 1 - type: int + expressions: key (type: string), 3 (type: int) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - null-subquery1-subquery2:a-subquery1-subquery2:inputtbl1 TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: 2 - type: int + expressions: key (type: string), 1 (type: int) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - null-subquery2:a-subquery2:inputtbl1 TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: 3 - type: int + expressions: key (type: string), 2 (type: int) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 90 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/union_remove_4.q.out ql/src/test/results/clientpositive/union_remove_4.q.out index 6b7437d..817856e 100644 --- ql/src/test/results/clientpositive/union_remove_4.q.out +++ ql/src/test/results/clientpositive/union_remove_4.q.out @@ -52,9 +52,6 @@ FROM ( SELECT key, count(1) as values from inputTbl1 group by key ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 @@ -68,56 +65,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -145,12 +126,10 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -159,12 +138,10 @@ STAGE PLANS: Stage: Stage-4 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -179,63 +156,46 @@ STAGE PLANS: Stage: Stage-7 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_5.q.out ql/src/test/results/clientpositive/union_remove_5.q.out index e5722f9..824d5bb 100644 --- ql/src/test/results/clientpositive/union_remove_5.q.out +++ ql/src/test/results/clientpositive/union_remove_5.q.out @@ -60,9 +60,6 @@ FROM ( SELECT key, 2 as values from inputTbl1 ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 values))))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 values))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-6 depends on stages: Stage-1, Stage-7, Stage-8 , consists of Stage-3, Stage-2, Stage-4 @@ -77,20 +74,17 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: UDFToLong(2) - type: bigint + expressions: key (type: string), UDFToLong(2) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -118,12 +112,10 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -132,12 +124,10 @@ STAGE PLANS: Stage: Stage-4 Map Reduce - Alias -> Map Operator Tree: -#### A masked pattern was here #### + Map Operator Tree: TableScan File Output Operator compressed: false - GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -152,56 +142,40 @@ STAGE PLANS: Stage: Stage-7 Map Reduce - Alias -> Map Operator Tree: - null-subquery1-subquery1:a-subquery1-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -210,27 +184,23 @@ STAGE PLANS: Stage: Stage-8 Map Reduce - Alias -> Map Operator Tree: - null-subquery1-subquery2:a-subquery1-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: UDFToLong(1) - type: bigint + expressions: key (type: string), UDFToLong(1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_7.q.out ql/src/test/results/clientpositive/union_remove_7.q.out index de53880..62b7dd5 100644 --- ql/src/test/results/clientpositive/union_remove_7.q.out +++ ql/src/test/results/clientpositive/union_remove_7.q.out @@ -56,9 +56,6 @@ FROM ( SELECT key, count(1) as values from inputTbl1 group by key ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2 @@ -67,56 +64,40 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -135,63 +116,46 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_8.q.out ql/src/test/results/clientpositive/union_remove_8.q.out index 5e8807e..8f0b378 100644 --- ql/src/test/results/clientpositive/union_remove_8.q.out +++ ql/src/test/results/clientpositive/union_remove_8.q.out @@ -62,9 +62,6 @@ FROM ( SELECT key, 2 as values from inputTbl1 ) a POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 values))))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 values))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1, Stage-2, Stage-3 @@ -74,20 +71,17 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:a-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: UDFToLong(2) - type: bigint + expressions: key (type: string), UDFToLong(2) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -106,56 +100,40 @@ STAGE PLANS: Stage: Stage-2 Map Reduce - Alias -> Map Operator Tree: - null-subquery1-subquery1:a-subquery1-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -164,27 +142,23 @@ STAGE PLANS: Stage: Stage-3 Map Reduce - Alias -> Map Operator Tree: - null-subquery1-subquery2:a-subquery1-subquery2:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: UDFToLong(1) - type: bigint + expressions: key (type: string), UDFToLong(1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_9.q.out ql/src/test/results/clientpositive/union_remove_9.q.out index b86eb02..28ca8da 100644 --- ql/src/test/results/clientpositive/union_remove_9.q.out +++ ql/src/test/results/clientpositive/union_remove_9.q.out @@ -64,9 +64,6 @@ select * FROM ( ) a )b POSTHOOK: type: QUERY -ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1) values)) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 1 values)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME inputTbl1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR 2 values))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))) b)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME outputTbl1))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) - STAGE DEPENDENCIES: Stage-1 is a root stage Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 @@ -80,54 +77,44 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Map Reduce - Alias -> Map Operator Tree: - null-subquery2:b-subquery2-subquery1:a-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: 1 - type: int + expressions: key (type: string), 1 (type: int) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - null-subquery2:b-subquery2-subquery2:a-subquery2:inputtbl1 TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string - expr: 2 - type: int + expressions: key (type: string), 2 (type: int) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Union + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint + expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 60 Basic stats: PARTIAL Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -167,63 +154,46 @@ STAGE PLANS: Stage: Stage-7 Map Reduce - Alias -> Map Operator Tree: - null-subquery1:b-subquery1:inputtbl1 + Map Operator Tree: TableScan alias: inputtbl1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: - expr: key - type: string + expressions: key (type: string) outputColumnNames: key + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: key - type: string + aggregations: count(1) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: - expr: _col0 - type: string + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: - expr: _col0 - type: string - tag: -1 - value expressions: - expr: _col1 - type: bigint + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator - aggregations: - expr: count(VALUE._col0) - bucketGroup: false - keys: - expr: KEY._col0 - type: string + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM (