Index: conf/hive-default.xml.template =================================================================== --- conf/hive-default.xml.template (revision 1584069) +++ conf/hive-default.xml.template (working copy) @@ -881,7 +881,7 @@ hive.auto.convert.join.use.nonstaged - true + false For conditional joins, if input stream from a small alias can be directly applied to join operator without filtering or projection, the alias need not to be pre-staged in distributed cache via mapred local task. Currently, this is not working with vectorization or tez execution engine. Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1584069) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -548,7 +548,7 @@ HIVECONVERTJOINNOCONDITIONALTASK("hive.auto.convert.join.noconditionaltask", true), HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD("hive.auto.convert.join.noconditionaltask.size", 10000000L), - HIVECONVERTJOINUSENONSTAGED("hive.auto.convert.join.use.nonstaged", true), + HIVECONVERTJOINUSENONSTAGED("hive.auto.convert.join.use.nonstaged", false), HIVESKEWJOINKEY("hive.skewjoin.key", 100000), HIVESKEWJOINMAPJOINNUMMAPTASK("hive.skewjoin.mapjoin.map.tasks", 10000), HIVESKEWJOINMAPJOINMINSPLIT("hive.skewjoin.mapjoin.min.split", 33554432L), //32M Index: ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out =================================================================== --- ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out (revision 1584069) +++ ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out (working copy) @@ -63,10 +63,30 @@ POSTHOOK: Lineage: table_desc.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: table_desc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -97,15 +117,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator Index: ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out =================================================================== --- ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out (revision 1584069) +++ ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out (working copy) @@ -154,10 +154,30 @@ on a.key=b.key and a.ds="2008-04-08" and b.ds="2008-04-08" POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -188,15 +208,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator Index: ql/src/test/results/clientpositive/mapjoin_memcheck.q.out =================================================================== --- ql/src/test/results/clientpositive/mapjoin_memcheck.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/mapjoin_memcheck.q.out (working copy) @@ -26,10 +26,30 @@ POSTHOOK: Lineage: src0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-5 is a root stage + Stage-2 depends on stages: Stage-5 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -58,15 +78,6 @@ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/join29.q.out =================================================================== --- ql/src/test/results/clientpositive/join29.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join29.q.out (working copy) @@ -23,11 +23,13 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1, Stage-4 , consists of Stage-5, Stage-6, Stage-2 - Stage-5 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-1, Stage-4 , consists of Stage-8, Stage-9, Stage-2 + Stage-8 has a backup stage: Stage-2 + Stage-5 depends on stages: Stage-8 Stage-0 depends on stages: Stage-2, Stage-5, Stage-6 Stage-3 depends on stages: Stage-0 - Stage-6 has a backup stage: Stage-2 + Stage-9 has a backup stage: Stage-2 + Stage-6 depends on stages: Stage-9 Stage-2 Stage-4 is a root stage @@ -75,6 +77,23 @@ Stage: Stage-7 Conditional Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -101,13 +120,6 @@ name: default.dest_j1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-0 Move Operator @@ -122,6 +134,23 @@ Stage: Stage-3 Stats-Aggr Operator + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -148,13 +177,6 @@ name: default.dest_j1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-2 Map Reduce Index: ql/src/test/results/clientpositive/auto_join22.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join22.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join22.q.out (working copy) @@ -5,10 +5,44 @@ SELECT sum(hash(src5.src1_value)) FROM (SELECT src3.*, src4.value as src4_value, src4.key as src4_key FROM src src4 JOIN (SELECT src2.*, src1.key as src1_key, src1.value as src1_value FROM src src1 JOIN src src2 ON src1.key = src2.key) src3 ON src3.src1_key = src4.key) src5 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-3 is a root stage + Stage-8 is a root stage + Stage-3 depends on stages: Stage-8 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + src5:src3:src1 + Fetch Operator + limit: -1 + src5:src4 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src5:src3:src1 + TableScan + alias: src1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + src5:src4 + TableScan + alias: src4 + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {_col3} + keys: + 0 key (type: string) + 1 _col2 (type: string) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -56,22 +90,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src5:src3:src1 - Fetch Operator - limit: -1 - src5:src4 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src5:src3:src1 - TableScan - alias: src1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - src5:src4 - TableScan - alias: src4 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) Index: ql/src/test/results/clientpositive/bucketmapjoin5.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin5.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin5.q.out (working copy) @@ -219,7 +219,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -229,6 +230,34 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket23.txt=[srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -281,23 +310,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket23.txt=[srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -800,7 +812,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -810,6 +823,34 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket23.txt=[srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -867,23 +908,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket23.txt=[srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/auto_join31.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join31.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join31.q.out (working copy) @@ -22,10 +22,12 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-8 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-6, Stage-7, Stage-2 - Stage-6 has a backup stage: Stage-2 + Stage-8 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-9, Stage-10, Stage-2 + Stage-9 has a backup stage: Stage-2 + Stage-6 depends on stages: Stage-9 Stage-3 depends on stages: Stage-2, Stage-6, Stage-7 - Stage-7 has a backup stage: Stage-2 + Stage-10 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-10 Stage-2 Stage-4 is a root stage Stage-5 is a root stage @@ -60,6 +62,39 @@ Stage: Stage-8 Conditional Operator + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -92,18 +127,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan - $INTNAME1 - TableScan Stage: Stage-3 Map Reduce @@ -131,6 +154,39 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + $INTNAME2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + $INTNAME2 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -163,18 +219,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - $INTNAME2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan - $INTNAME2 - TableScan Stage: Stage-2 Map Reduce Index: ql/src/test/results/clientpositive/join33.q.out =================================================================== --- ql/src/test/results/clientpositive/join33.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join33.q.out (working copy) @@ -95,11 +95,95 @@ STAGE DEPENDENCIES: - Stage-5 is a root stage + Stage-7 is a root stage + Stage-5 depends on stages: Stage-7 Stage-0 depends on stages: Stage-5 Stage-2 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + z + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + z + TableScan + alias: z + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col5} {_col0} + 1 {value} + keys: + 0 _col1 (type: string) + 1 value (type: string) + Position of Big Table: 0 + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -163,70 +247,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - z - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 11 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 0 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 0 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - z - TableScan - alias: z - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out =================================================================== --- ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out (working copy) @@ -7,10 +7,44 @@ explain select srcpart.key from srcpart join src on (srcpart.value=src.value) join src1 on (srcpart.key=src1.key) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-5 is a root stage + Stage-7 is a root stage + Stage-5 depends on stages: Stage-7 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + src + Fetch Operator + limit: -1 + src1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src + TableScan + alias: src + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 value (type: string) + 1 value (type: string) + src1 + TableScan + alias: src1 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -52,22 +86,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src - Fetch Operator - limit: -1 - src1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src - TableScan - alias: src - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - src1 - TableScan - alias: src1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -91,6 +109,9 @@ src Fetch Operator limit: -1 + src1 + Fetch Operator + limit: -1 Alias -> Map Local Operator Tree: src TableScan @@ -106,6 +127,17 @@ keys: 0 value (type: string) 1 value (type: string) + src1 + TableScan + alias: src1 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) Stage: Stage-5 Map Reduce @@ -151,17 +183,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src - src1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src - src1 - TableScan - alias: src1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -214,11 +235,32 @@ select count(*) from srcpart join src on (srcpart.value=src.value) join src src1 on (srcpart.key=src1.key) group by ds POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-3 depends on stages: Stage-7 + Stage-9 is a root stage + Stage-7 depends on stages: Stage-9 + Stage-8 depends on stages: Stage-7 + Stage-3 depends on stages: Stage-8 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + src + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src + TableScan + alias: src + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {ds} + 1 + keys: + 0 value (type: string) + 1 value (type: string) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -244,16 +286,26 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src - TableScan - alias: src - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col2} + 1 + keys: + 0 _col0 (type: string) + 1 key (type: string) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -287,15 +339,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) Index: ql/src/test/results/clientpositive/auto_join9.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join9.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join9.q.out (working copy) @@ -14,11 +14,31 @@ INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where src1.ds = '2008-04-08' and src1.hr = '12' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage + Stage-5 is a root stage + Stage-4 depends on stages: Stage-5 Stage-0 depends on stages: Stage-4 Stage-2 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -50,15 +70,6 @@ name: default.dest1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Move Operator Index: ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out (working copy) @@ -426,10 +426,12 @@ POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-9 depends on stages: Stage-1, Stage-5 , consists of Stage-7, Stage-8, Stage-3 - Stage-7 has a backup stage: Stage-3 + Stage-9 depends on stages: Stage-1, Stage-5 , consists of Stage-10, Stage-11, Stage-3 + Stage-10 has a backup stage: Stage-3 + Stage-7 depends on stages: Stage-10 Stage-4 depends on stages: Stage-3, Stage-7, Stage-8 - Stage-8 has a backup stage: Stage-3 + Stage-11 has a backup stage: Stage-3 + Stage-8 depends on stages: Stage-11 Stage-3 Stage-5 is a root stage Stage-0 is a root stage @@ -483,6 +485,23 @@ Stage: Stage-9 Conditional Operator + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -508,13 +527,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-4 Map Reduce @@ -533,6 +545,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-11 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Stage: Stage-8 Map Reduce Map Operator Tree: @@ -558,13 +587,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-3 Map Reduce @@ -1724,9 +1746,11 @@ POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-4, Stage-1 - Stage-3 has a backup stage: Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-1 Stage-0 is a root stage @@ -1734,6 +1758,24 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + subq1:b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq1:b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -1758,14 +1800,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq1:b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq1:b - TableScan - alias: b Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1781,6 +1815,24 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + subq1:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq1:a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -1805,14 +1857,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq1:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq1:a - TableScan - alias: a Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1914,10 +1958,12 @@ POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-6 is a root stage , consists of Stage-4, Stage-5, Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-2 depends on stages: Stage-1, Stage-4, Stage-5 - Stage-5 has a backup stage: Stage-1 + Stage-8 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-8 Stage-1 Stage-0 is a root stage @@ -1925,6 +1971,24 @@ Stage: Stage-6 Conditional Operator + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + subq1:b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq1:b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -1955,14 +2019,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq1:b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq1:b - TableScan - alias: b Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1996,6 +2052,24 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + subq1:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq1:a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -2026,14 +2100,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq1:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq1:a - TableScan - alias: a Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2156,10 +2222,12 @@ POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-6 is a root stage , consists of Stage-4, Stage-5, Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-3 depends on stages: Stage-1, Stage-4, Stage-5 - Stage-5 has a backup stage: Stage-1 + Stage-8 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-8 Stage-1 Stage-0 is a root stage @@ -2167,6 +2235,24 @@ Stage: Stage-6 Conditional Operator + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + subq2:subq1:b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq2:subq1:b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -2197,14 +2283,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq2:subq1:b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq2:subq1:b - TableScan - alias: b Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2245,6 +2323,24 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + subq2:subq1:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq2:subq1:a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -2275,14 +2371,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq2:subq1:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq2:subq1:a - TableScan - alias: a Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2426,18 +2514,24 @@ POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-12 is a root stage , consists of Stage-10, Stage-11, Stage-1 - Stage-10 has a backup stage: Stage-1 - Stage-9 depends on stages: Stage-1, Stage-5, Stage-10, Stage-11, Stage-13, Stage-14 , consists of Stage-7, Stage-8, Stage-3 - Stage-7 has a backup stage: Stage-3 + Stage-12 is a root stage , consists of Stage-18, Stage-19, Stage-1 + Stage-18 has a backup stage: Stage-1 + Stage-10 depends on stages: Stage-18 + Stage-9 depends on stages: Stage-1, Stage-5, Stage-10, Stage-11, Stage-13, Stage-14 , consists of Stage-16, Stage-17, Stage-3 + Stage-16 has a backup stage: Stage-3 + Stage-7 depends on stages: Stage-16 Stage-4 depends on stages: Stage-3, Stage-7, Stage-8 - Stage-8 has a backup stage: Stage-3 + Stage-17 has a backup stage: Stage-3 + Stage-8 depends on stages: Stage-17 Stage-3 - Stage-11 has a backup stage: Stage-1 + Stage-19 has a backup stage: Stage-1 + Stage-11 depends on stages: Stage-19 Stage-1 - Stage-15 is a root stage , consists of Stage-13, Stage-14, Stage-5 - Stage-13 has a backup stage: Stage-5 - Stage-14 has a backup stage: Stage-5 + Stage-15 is a root stage , consists of Stage-20, Stage-21, Stage-5 + Stage-20 has a backup stage: Stage-5 + Stage-13 depends on stages: Stage-20 + Stage-21 has a backup stage: Stage-5 + Stage-14 depends on stages: Stage-21 Stage-5 Stage-0 is a root stage @@ -2445,6 +2539,24 @@ Stage: Stage-12 Conditional Operator + Stage: Stage-18 + Map Reduce Local Work + Alias -> Map Local Tables: + src1:subq1:b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src1:subq1:b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-10 Map Reduce Map Operator Tree: @@ -2475,14 +2587,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1:subq1:b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1:subq1:b - TableScan - alias: b Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2502,6 +2606,23 @@ Stage: Stage-9 Conditional Operator + Stage: Stage-16 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -2527,13 +2648,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-4 Map Reduce @@ -2552,6 +2666,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-17 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Stage: Stage-8 Map Reduce Map Operator Tree: @@ -2577,13 +2708,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-3 Map Reduce @@ -2618,6 +2742,24 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-19 + Map Reduce Local Work + Alias -> Map Local Tables: + src1:subq1:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src1:subq1:a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-11 Map Reduce Map Operator Tree: @@ -2648,14 +2790,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1:subq1:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1:subq1:a - TableScan - alias: a Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2720,6 +2854,24 @@ Stage: Stage-15 Conditional Operator + Stage: Stage-20 + Map Reduce Local Work + Alias -> Map Local Tables: + src2:subq2:b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src2:subq2:b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-13 Map Reduce Map Operator Tree: @@ -2750,14 +2902,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src2:subq2:b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src2:subq2:b - TableScan - alias: b Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2774,6 +2918,24 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-21 + Map Reduce Local Work + Alias -> Map Local Tables: + src2:subq2:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src2:subq2:a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-14 Map Reduce Map Operator Tree: @@ -2804,14 +2966,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src2:subq2:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src2:subq2:a - TableScan - alias: a Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -3201,8 +3355,9 @@ POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-7, Stage-1 - Stage-3 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 Stage-7 has a backup stage: Stage-1 Stage-4 depends on stages: Stage-7 Stage-1 @@ -3212,6 +3367,24 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -3241,14 +3414,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -3957,8 +4122,9 @@ POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-7, Stage-1 - Stage-3 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 Stage-7 has a backup stage: Stage-1 Stage-4 depends on stages: Stage-7 Stage-1 @@ -3968,6 +4134,24 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -3997,14 +4181,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -4171,10 +4347,11 @@ POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-6, Stage-4, Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 Stage-6 has a backup stage: Stage-1 Stage-3 depends on stages: Stage-6 - Stage-4 has a backup stage: Stage-1 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-1 Stage-0 is a root stage @@ -4244,6 +4421,24 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 _col0 (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -4273,14 +4468,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -4794,8 +4981,9 @@ POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-7, Stage-1 - Stage-3 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 Stage-7 has a backup stage: Stage-1 Stage-4 depends on stages: Stage-7 Stage-1 @@ -4805,6 +4993,24 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a:b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a:b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col0 (type: int) + 1 key (type: int) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -4834,14 +5040,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a:b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a:b - TableScan - alias: b Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) Index: ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out (working copy) @@ -259,17 +259,37 @@ POSTHOOK: Lineage: test_table3 PARTITION(ds=1).key2 SIMPLE [(test_table1)a.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: test_table3 PARTITION(ds=1).value EXPRESSION [(test_table1)a.FieldSchema(name:value, type:string, comment:null), (test_table2)b.FieldSchema(name:value, type:string, comment:null), ] STAGE DEPENDENCIES: - Stage-6 is a root stage , consists of Stage-4, Stage-5, Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-0 depends on stages: Stage-1, Stage-4, Stage-5 Stage-3 depends on stages: Stage-0 - Stage-5 has a backup stage: Stage-1 + Stage-8 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-8 Stage-1 STAGE PLANS: Stage: Stage-6 Conditional Operator + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -295,14 +315,6 @@ value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b Reduce Operator Tree: Extract File Output Operator @@ -328,6 +340,24 @@ Stage: Stage-3 Stats-Aggr Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -353,14 +383,6 @@ value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a Reduce Operator Tree: Extract File Output Operator Index: ql/src/test/results/clientpositive/multi_join_union.q.out =================================================================== --- ql/src/test/results/clientpositive/multi_join_union.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/multi_join_union.q.out (working copy) @@ -45,6 +45,9 @@ Stage: Stage-8 Map Reduce Local Work Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 c-subquery1:a-subquery1:src13 Fetch Operator limit: -1 @@ -52,6 +55,17 @@ Fetch Operator limit: -1 Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) c-subquery1:a-subquery1:src13 TableScan alias: src13 @@ -136,19 +150,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - c-subquery1:a-subquery1:src13 - c-subquery2:a-subquery2:src14 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - c-subquery1:a-subquery1:src13 - c-subquery2:a-subquery2:src14 Stage: Stage-0 Fetch Operator Index: ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out (working copy) @@ -478,9 +478,11 @@ STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-4, Stage-1 - Stage-3 has a backup stage: Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-1 Stage-0 is a root stage @@ -488,6 +490,119 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 226 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 226 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -515,108 +630,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 226 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 226 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -794,6 +807,74 @@ GatherStats: false MultiFileSpray: false + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=2008-04-08 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_big + name: default.bucket_big + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -821,63 +902,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: ds=2008-04-08 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_big - name: default.bucket_big - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/correlationoptimizer3.q.out =================================================================== --- ql/src/test/results/clientpositive/correlationoptimizer3.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/correlationoptimizer3.q.out (working copy) @@ -453,11 +453,45 @@ ON b.key = d.key) tmp POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-9 is a root stage + Stage-2 depends on stages: Stage-9 Stage-3 depends on stages: Stage-2 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + tmp:b:x + Fetch Operator + limit: -1 + tmp:d:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + tmp:b:x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + tmp:d:x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -517,22 +551,6 @@ value expressions: _col1 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - tmp:b:x - Fetch Operator - limit: -1 - tmp:d:x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - tmp:b:x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - tmp:d:x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE @@ -1087,11 +1105,45 @@ ON b.key = d.key) tmp POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-9 is a root stage + Stage-2 depends on stages: Stage-9 Stage-3 depends on stages: Stage-2 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + tmp:b:x + Fetch Operator + limit: -1 + tmp:d:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + tmp:b:x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + tmp:d:x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -1151,22 +1203,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - tmp:b:x - Fetch Operator - limit: -1 - tmp:d:x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - tmp:b:x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - tmp:d:x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/bucketcontext_7.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketcontext_7.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketcontext_7.q.out (working copy) @@ -156,10 +156,132 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket4outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket3outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket4outof4.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -193,116 +315,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket4outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket3outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket4outof4.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out (working copy) @@ -136,10 +136,79 @@ STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-5 is a root stage + Stage-2 depends on stages: Stage-5 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=2008-04-08 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 114 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -172,63 +241,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: ds=2008-04-08 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 114 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -475,10 +487,79 @@ STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-5 is a root stage + Stage-2 depends on stages: Stage-5 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=2008-04-08 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 114 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -511,63 +592,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: ds=2008-04-08 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 114 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -813,10 +837,85 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 114 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket3outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket4outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -850,69 +949,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 114 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket3outof4.txt=[ds=2008-04-08/smallsrcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket4outof4.txt=[ds=2008-04-08/smallsrcsortbucket2outof4.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/infer_bucket_sort_convert_join.q.out =================================================================== --- ql/src/test/results/clientpositive/infer_bucket_sort_convert_join.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/infer_bucket_sort_convert_join.q.out (working copy) @@ -76,17 +76,17 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@test_table@part=1 -Execution failed with exit status: 2 +Execution failed with exit status: 3 Obtaining error information Task failed! Task ID: - Stage-4 + Stage-7 Logs: #### A masked pattern was here #### -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask +FAILED: Execution Error, return code 3 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask POSTHOOK: query: -- This test tests the scenario when the mapper dies. So, create a conditional task for the mapjoin. -- Tests a join which is not converted to a map join, the output should be bucketed and sorted. Index: ql/src/test/results/clientpositive/auto_join29.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join29.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join29.q.out (working copy) @@ -13,10 +13,30 @@ Stage: Stage-6 Map Reduce Local Work Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 src2 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 {(key < 10)} + 1 + 2 {(key < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) src2 TableScan alias: src2 @@ -26,13 +46,17 @@ Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: - 0 + 0 {key} {value} 1 {key} {value} 2 {key} {value} filter predicates: - 0 + 0 {(key < 10)} 1 2 {(key < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) Stage: Stage-2 Map Reduce @@ -69,17 +93,6 @@ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - src2 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - src2 Reduce Operator Tree: Extract Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE @@ -1875,6 +1888,9 @@ src1 Fetch Operator limit: -1 + src2 + Fetch Operator + limit: -1 Alias -> Map Local Operator Tree: src1 TableScan @@ -1886,12 +1902,33 @@ HashTable Sink Operator condition expressions: 0 {key} {value} - 1 + 1 {key} {value} 2 {key} {value} filter predicates: 0 - 1 + 1 {(key > 10)} 2 {(key < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + src2 + TableScan + alias: src2 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 + 1 {(key > 10)} + 2 {(key < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) Stage: Stage-2 Map Reduce @@ -1928,17 +1965,6 @@ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - src2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1 - src2 - TableScan - alias: src2 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/reduce_deduplicate_exclude_join.q.out =================================================================== --- ql/src/test/results/clientpositive/reduce_deduplicate_exclude_join.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/reduce_deduplicate_exclude_join.q.out (working copy) @@ -4,7 +4,8 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-3 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-4 Stage-0 is a root stage STAGE PLANS: @@ -34,6 +35,25 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {key} {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -65,15 +85,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator Index: ql/src/test/results/clientpositive/bucketcontext_2.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketcontext_2.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketcontext_2.q.out (working copy) @@ -123,10 +123,86 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -160,70 +236,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/auto_join15.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join15.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join15.q.out (working copy) @@ -15,11 +15,31 @@ ) a POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 Stage-3 depends on stages: Stage-2 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a:src1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a:src1 + TableScan + alias: src1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -48,15 +68,6 @@ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a:src1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a:src1 - TableScan - alias: src1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/auto_join24.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join24.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join24.q.out (working copy) @@ -26,10 +26,30 @@ POSTHOOK: Lineage: tst1.cnt EXPRESSION [(src)a.null, ] POSTHOOK: Lineage: tst1.key SIMPLE [(src)a.FieldSchema(name:key, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-5 is a root stage + Stage-2 depends on stages: Stage-5 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {cnt} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -62,15 +82,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) Index: ql/src/test/results/clientpositive/bucketmapjoin7.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin7.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin7.q.out (working copy) @@ -125,10 +125,86 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 0 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {ds=2008-04-08/hr=0/srcbucket20.txt=[ds=2008-04-08/hr=0/srcbucket20.txt], ds=2008-04-08/hr=0/srcbucket21.txt=[ds=2008-04-08/hr=0/srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -179,70 +255,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 0 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {ds=2008-04-08/hr=0/srcbucket20.txt=[ds=2008-04-08/hr=0/srcbucket20.txt], ds=2008-04-08/hr=0/srcbucket21.txt=[ds=2008-04-08/hr=0/srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketmapjoin11.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin11.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin11.q.out (working copy) @@ -200,10 +200,130 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 4 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 2 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 2140 Data size: 8562 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt, part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt, part=1/srcbucket23.txt, part=2/srcbucket21.txt], part=2/srcbucket20.txt=[part=1/srcbucket20.txt, part=2/srcbucket20.txt], part=2/srcbucket21.txt=[part=1/srcbucket21.txt, part=2/srcbucket21.txt], part=2/srcbucket22.txt=[part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=2/srcbucket23.txt=[part=1/srcbucket23.txt, part=2/srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -237,114 +357,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 4 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 2 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 2140 Data size: 8562 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt, part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt, part=1/srcbucket23.txt, part=2/srcbucket21.txt], part=2/srcbucket20.txt=[part=1/srcbucket20.txt, part=2/srcbucket20.txt], part=2/srcbucket21.txt=[part=1/srcbucket21.txt, part=2/srcbucket21.txt], part=2/srcbucket22.txt=[part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=2/srcbucket23.txt=[part=1/srcbucket23.txt, part=2/srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -577,10 +589,130 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 4 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 2 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 2140 Data size: 8562 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int), part (type: string) + 1 key (type: int), part (type: string) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt, part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt, part=1/srcbucket23.txt, part=2/srcbucket21.txt], part=2/srcbucket20.txt=[part=1/srcbucket20.txt, part=2/srcbucket20.txt], part=2/srcbucket21.txt=[part=1/srcbucket21.txt, part=2/srcbucket21.txt], part=2/srcbucket22.txt=[part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=2/srcbucket23.txt=[part=1/srcbucket23.txt, part=2/srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -614,114 +746,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 4 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 2 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 2140 Data size: 8562 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt, part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt, part=1/srcbucket23.txt, part=2/srcbucket21.txt], part=2/srcbucket20.txt=[part=1/srcbucket20.txt, part=2/srcbucket20.txt], part=2/srcbucket21.txt=[part=1/srcbucket21.txt, part=2/srcbucket21.txt], part=2/srcbucket22.txt=[part=1/srcbucket22.txt, part=2/srcbucket20.txt], part=2/srcbucket23.txt=[part=1/srcbucket23.txt, part=2/srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/join26.q.out =================================================================== --- ql/src/test/results/clientpositive/join26.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join26.q.out (working copy) @@ -97,7 +97,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-10 is a root stage + Stage-1 depends on stages: Stage-10 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -107,6 +108,47 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 + y + TableScan + alias: y + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -161,24 +203,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - y - TableScan - alias: y - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out (working copy) @@ -130,7 +130,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6 Stage-5 Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 @@ -140,6 +141,73 @@ Stage-7 depends on stages: Stage-6 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 3 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part + numFiles 3 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 4200 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 3 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part + partition_columns ds + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part + name: default.srcbucket_mapjoin_part + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 40 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -191,62 +259,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 3 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part - numFiles 3 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 4200 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 3 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part - partition_columns ds - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part - name: default.srcbucket_mapjoin_part - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 40 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/join35.q.out =================================================================== --- ql/src/test/results/clientpositive/join35.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join35.q.out (working copy) @@ -143,7 +143,8 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-4 + Stage-7 depends on stages: Stage-1, Stage-4 + Stage-6 depends on stages: Stage-7 Stage-0 depends on stages: Stage-6 Stage-3 depends on stages: Stage-0 Stage-4 is a root stage @@ -256,6 +257,27 @@ GatherStats: false MultiFileSpray: false + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col1} + 1 {key} {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + Position of Big Table: 0 + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -353,16 +375,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketmapjoin2.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin2.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin2.q.out (working copy) @@ -153,7 +153,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6 Stage-5 Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 @@ -163,6 +164,80 @@ Stage-7 depends on stages: Stage-6 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 3062 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns ds + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {ds=2008-04-08/srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt], ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket23.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -215,69 +290,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 3062 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns ds - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {ds=2008-04-08/srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt], ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket23.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -739,7 +751,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -749,6 +762,80 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part + partition_columns ds + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part + name: default.srcbucket_mapjoin_part + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -806,69 +893,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part - partition_columns ds - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part - name: default.srcbucket_mapjoin_part - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1521,7 +1545,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6 Stage-5 Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 @@ -1531,6 +1556,125 @@ Stage-7 depends on stages: Stage-6 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 3062 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns ds + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 3062 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns ds + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 58 Data size: 6124 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {ds=2008-04-08/srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt, ds=2008-04-09/srcbucket22.txt], ds=2008-04-08/srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt, ds=2008-04-09/srcbucket23.txt], ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket22.txt, ds=2008-04-09/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket23.txt, ds=2008-04-09/srcbucket23.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -1588,114 +1732,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 3062 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns ds - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 3062 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns ds - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 58 Data size: 6124 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {ds=2008-04-08/srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt, ds=2008-04-09/srcbucket22.txt], ds=2008-04-08/srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt, ds=2008-04-09/srcbucket23.txt], ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket22.txt, ds=2008-04-09/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket23.txt, ds=2008-04-09/srcbucket23.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/mapjoin_subquery2.q.out =================================================================== --- ql/src/test/results/clientpositive/mapjoin_subquery2.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/mapjoin_subquery2.q.out (working copy) @@ -80,10 +80,44 @@ JOIN z ON (subq.key1 = z.id) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-5 is a root stage + Stage-7 is a root stage + Stage-5 depends on stages: Stage-7 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + subq:y + Fetch Operator + limit: -1 + z + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq:y + TableScan + alias: y + Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {id} {name} + 1 {name} {id} + keys: + 0 id (type: int) + 1 id (type: int) + z + TableScan + alias: z + Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} + 1 {id} {name} + keys: + 0 _col0 (type: int) + 1 id (type: int) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -129,22 +163,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq:y - Fetch Operator - limit: -1 - z - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq:y - TableScan - alias: y - Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE - z - TableScan - alias: z - Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE Stage: Stage-0 Fetch Operator Index: ql/src/test/results/clientpositive/mapjoin1.q.out =================================================================== --- ql/src/test/results/clientpositive/mapjoin1.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/mapjoin1.q.out (working copy) @@ -30,10 +30,33 @@ SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + filter predicates: + 0 + 1 {true} + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -70,15 +93,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -111,10 +125,33 @@ SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + filter predicates: + 0 + 1 {((key * 10) < '1000')} + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -151,15 +188,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -194,10 +222,33 @@ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {_col0} {_col1} + filter predicates: + 0 + 1 {(_col1.key > 200)} + keys: + 0 key (type: string) + 1 _col0 (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -238,15 +289,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -467,10 +509,30 @@ (select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {_col0} {_col1} + keys: + 0 key (type: string) + 1 _col0 (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -511,15 +573,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator Index: ql/src/test/results/clientpositive/join_map_ppr.q.out =================================================================== --- ql/src/test/results/clientpositive/join_map_ppr.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join_map_ppr.q.out (working copy) @@ -99,7 +99,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-10 is a root stage + Stage-1 depends on stages: Stage-10 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -109,6 +110,47 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 + y + TableScan + alias: y + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -163,24 +205,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - y - TableScan - alias: y - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -704,7 +728,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-10 is a root stage + Stage-1 depends on stages: Stage-10 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -714,6 +739,47 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + 2 {value} + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Position of Big Table: 2 + y + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + 2 {value} + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Position of Big Table: 2 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -773,24 +839,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - y - TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/join30.q.out =================================================================== --- ql/src/test/results/clientpositive/join30.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join30.q.out (working copy) @@ -14,11 +14,31 @@ SELECT /*+ MAPJOIN(x) */ x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-4 is a root stage + Stage-1 depends on stages: Stage-4 Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -54,15 +74,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) Index: ql/src/test/results/clientpositive/mapjoin_hook.q.out =================================================================== --- ql/src/test/results/clientpositive/mapjoin_hook.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/mapjoin_hook.q.out (working copy) @@ -11,7 +11,8 @@ PREHOOK: Input: default@src PREHOOK: Input: default@src1 PREHOOK: Output: default@dest1 -[MapJoinCounter PostHook] COMMON_JOIN: 0 HINTED_MAPJOIN: 1 HINTED_MAPJOIN_LOCAL: 0 CONVERTED_MAPJOIN: 0 CONVERTED_MAPJOIN_LOCAL: 0 BACKUP_COMMON_JOIN: 0 +[MapJoinCounter PostHook] COMMON_JOIN: 0 HINTED_MAPJOIN: 1 HINTED_MAPJOIN_LOCAL: 1 CONVERTED_MAPJOIN: 0 CONVERTED_MAPJOIN_LOCAL: 0 BACKUP_COMMON_JOIN: 0 +RUN: Stage-4:MAPREDLOCAL RUN: Stage-1:MAPRED RUN: Stage-0:MOVE RUN: Stage-2:STATS @@ -20,7 +21,8 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@dest1 -[MapJoinCounter PostHook] COMMON_JOIN: 0 HINTED_MAPJOIN: 1 HINTED_MAPJOIN_LOCAL: 0 CONVERTED_MAPJOIN: 0 CONVERTED_MAPJOIN_LOCAL: 0 BACKUP_COMMON_JOIN: 0 +[MapJoinCounter PostHook] COMMON_JOIN: 0 HINTED_MAPJOIN: 1 HINTED_MAPJOIN_LOCAL: 1 CONVERTED_MAPJOIN: 0 CONVERTED_MAPJOIN_LOCAL: 0 BACKUP_COMMON_JOIN: 0 +RUN: Stage-6:MAPREDLOCAL RUN: Stage-5:MAPRED RUN: Stage-0:MOVE RUN: Stage-2:STATS @@ -35,21 +37,21 @@ PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@dest1 -Execution failed with exit status: 2 +Execution failed with exit status: 3 Obtaining error information Task failed! Task ID: - Stage-4 + Stage-7 Logs: #### A masked pattern was here #### -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask +FAILED: Execution Error, return code 3 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask -[MapJoinCounter PostHook] COMMON_JOIN: 0 HINTED_MAPJOIN: 0 HINTED_MAPJOIN_LOCAL: 0 CONVERTED_MAPJOIN: 1 CONVERTED_MAPJOIN_LOCAL: 0 BACKUP_COMMON_JOIN: 1 +[MapJoinCounter PostHook] COMMON_JOIN: 0 HINTED_MAPJOIN: 0 HINTED_MAPJOIN_LOCAL: 0 CONVERTED_MAPJOIN: 0 CONVERTED_MAPJOIN_LOCAL: 1 BACKUP_COMMON_JOIN: 1 RUN: Stage-6:CONDITIONAL -RUN: Stage-4:MAPRED +RUN: Stage-7:MAPREDLOCAL RUN: Stage-1:MAPRED RUN: Stage-0:MOVE RUN: Stage-2:STATS @@ -58,36 +60,36 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@dest1 -Execution failed with exit status: 2 +Execution failed with exit status: 3 Obtaining error information Task failed! Task ID: - Stage-10 + Stage-14 Logs: #### A masked pattern was here #### -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask +FAILED: Execution Error, return code 3 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask -Execution failed with exit status: 2 +Execution failed with exit status: 3 Obtaining error information Task failed! Task ID: - Stage-6 + Stage-12 Logs: #### A masked pattern was here #### -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask +FAILED: Execution Error, return code 3 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask -[MapJoinCounter PostHook] COMMON_JOIN: 0 HINTED_MAPJOIN: 0 HINTED_MAPJOIN_LOCAL: 0 CONVERTED_MAPJOIN: 2 CONVERTED_MAPJOIN_LOCAL: 0 BACKUP_COMMON_JOIN: 2 +[MapJoinCounter PostHook] COMMON_JOIN: 0 HINTED_MAPJOIN: 0 HINTED_MAPJOIN_LOCAL: 0 CONVERTED_MAPJOIN: 0 CONVERTED_MAPJOIN_LOCAL: 2 BACKUP_COMMON_JOIN: 2 RUN: Stage-11:CONDITIONAL -RUN: Stage-10:MAPRED +RUN: Stage-14:MAPREDLOCAL RUN: Stage-1:MAPRED RUN: Stage-8:CONDITIONAL -RUN: Stage-6:MAPRED +RUN: Stage-12:MAPREDLOCAL RUN: Stage-2:MAPRED RUN: Stage-0:MOVE RUN: Stage-3:STATS Index: ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out (working copy) @@ -427,17 +427,37 @@ POSTHOOK: Lineage: test_table3 PARTITION(ds=1).key2 SIMPLE [(test_table1)a.FieldSchema(name:key2, type:int, comment:null), ] POSTHOOK: Lineage: test_table3 PARTITION(ds=1).value EXPRESSION [(test_table1)a.FieldSchema(name:value, type:string, comment:null), (test_table2)b.FieldSchema(name:value, type:string, comment:null), ] STAGE DEPENDENCIES: - Stage-6 is a root stage , consists of Stage-4, Stage-5, Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-0 depends on stages: Stage-1, Stage-4, Stage-5 Stage-3 depends on stages: Stage-0 - Stage-5 has a backup stage: Stage-1 + Stage-8 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-8 Stage-1 STAGE PLANS: Stage: Stage-6 Conditional Operator + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -463,14 +483,6 @@ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b Reduce Operator Tree: Extract File Output Operator @@ -496,6 +508,24 @@ Stage: Stage-3 Stats-Aggr Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -521,14 +551,6 @@ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a Reduce Operator Tree: Extract File Output Operator @@ -608,17 +630,37 @@ POSTHOOK: Lineage: test_table3 PARTITION(ds=1).key2 SIMPLE [(test_table1)a.FieldSchema(name:key2, type:int, comment:null), ] POSTHOOK: Lineage: test_table3 PARTITION(ds=1).value EXPRESSION [(test_table1)a.FieldSchema(name:value, type:string, comment:null), (test_table2)b.FieldSchema(name:value, type:string, comment:null), ] STAGE DEPENDENCIES: - Stage-6 is a root stage , consists of Stage-4, Stage-5, Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-0 depends on stages: Stage-1, Stage-4, Stage-5 Stage-2 depends on stages: Stage-0 - Stage-5 has a backup stage: Stage-1 + Stage-8 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-8 Stage-1 STAGE PLANS: Stage: Stage-6 Conditional Operator + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + subq1:b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq1:b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -644,14 +686,6 @@ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq1:b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq1:b - TableScan - alias: b Reduce Operator Tree: Extract File Output Operator @@ -677,6 +711,24 @@ Stage: Stage-2 Stats-Aggr Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + subq1:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq1:a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -702,14 +754,6 @@ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq1:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq1:a - TableScan - alias: a Reduce Operator Tree: Extract File Output Operator @@ -1254,17 +1298,37 @@ POSTHOOK: Lineage: test_table3 PARTITION(ds=1).key2 SIMPLE [(test_table1)a.FieldSchema(name:key2, type:int, comment:null), ] POSTHOOK: Lineage: test_table3 PARTITION(ds=1).value EXPRESSION [(test_table1)a.FieldSchema(name:value, type:string, comment:null), (test_table2)b.FieldSchema(name:value, type:string, comment:null), ] STAGE DEPENDENCIES: - Stage-6 is a root stage , consists of Stage-4, Stage-5, Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-0 depends on stages: Stage-1, Stage-4, Stage-5 Stage-3 depends on stages: Stage-0 - Stage-5 has a backup stage: Stage-1 + Stage-8 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-8 Stage-1 STAGE PLANS: Stage: Stage-6 Conditional Operator + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + subq2:subq1:b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq2:subq1:b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -1290,14 +1354,6 @@ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq2:subq1:b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq2:subq1:b - TableScan - alias: b Reduce Operator Tree: Extract File Output Operator @@ -1323,6 +1379,24 @@ Stage: Stage-3 Stats-Aggr Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + subq2:subq1:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq2:subq1:a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {value} + keys: + 0 key (type: int), key2 (type: int) + 1 key (type: int), key2 (type: int) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -1348,14 +1422,6 @@ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq2:subq1:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq2:subq1:a - TableScan - alias: a Reduce Operator Tree: Extract File Output Operator Index: ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_6.q.out (working copy) @@ -455,11 +455,14 @@ POSTHOOK: Lineage: tbl4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-8 is a root stage , consists of Stage-5, Stage-6, Stage-7, Stage-1 - Stage-5 has a backup stage: Stage-1 + Stage-8 is a root stage , consists of Stage-9, Stage-10, Stage-11, Stage-1 + Stage-9 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-9 Stage-2 depends on stages: Stage-1, Stage-5, Stage-6, Stage-7 - Stage-6 has a backup stage: Stage-1 - Stage-7 has a backup stage: Stage-1 + Stage-10 has a backup stage: Stage-1 + Stage-6 depends on stages: Stage-10 + Stage-11 has a backup stage: Stage-1 + Stage-7 depends on stages: Stage-11 Stage-1 Stage-0 is a root stage @@ -467,6 +470,41 @@ Stage: Stage-8 Conditional Operator + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + c + TableScan + alias: c + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -497,20 +535,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - c - TableScan - alias: c Stage: Stage-2 Map Reduce @@ -538,6 +562,41 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + c + TableScan + alias: c + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -568,21 +627,42 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - c - TableScan - alias: c + Stage: Stage-11 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -613,20 +693,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - b - TableScan - alias: b Stage: Stage-1 Map Reduce @@ -1032,11 +1098,14 @@ POSTHOOK: Lineage: tbl4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-8 is a root stage , consists of Stage-5, Stage-6, Stage-7, Stage-1 - Stage-5 has a backup stage: Stage-1 + Stage-8 is a root stage , consists of Stage-9, Stage-10, Stage-11, Stage-1 + Stage-9 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-9 Stage-2 depends on stages: Stage-1, Stage-5, Stage-6, Stage-7 - Stage-6 has a backup stage: Stage-1 - Stage-7 has a backup stage: Stage-1 + Stage-10 has a backup stage: Stage-1 + Stage-6 depends on stages: Stage-10 + Stage-11 has a backup stage: Stage-1 + Stage-7 depends on stages: Stage-11 Stage-1 Stage-0 is a root stage @@ -1044,6 +1113,41 @@ Stage: Stage-8 Conditional Operator + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + c + TableScan + alias: c + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -1074,20 +1178,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - c - TableScan - alias: c Stage: Stage-2 Map Reduce @@ -1115,6 +1205,41 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + c + TableScan + alias: c + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -1145,21 +1270,42 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - c - TableScan - alias: c + Stage: Stage-11 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(key) (type: double) + 2 UDFToDouble(key) (type: double) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -1190,20 +1336,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - b - TableScan - alias: b Stage: Stage-1 Map Reduce @@ -1296,10 +1428,13 @@ POSTHOOK: Lineage: tbl4.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-7 is a root stage , consists of Stage-4, Stage-5, Stage-6, Stage-1 - Stage-4 has a backup stage: Stage-1 - Stage-5 has a backup stage: Stage-1 - Stage-6 has a backup stage: Stage-1 + Stage-7 is a root stage , consists of Stage-8, Stage-9, Stage-10, Stage-1 + Stage-8 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-8 + Stage-9 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-9 + Stage-10 has a backup stage: Stage-1 + Stage-6 depends on stages: Stage-10 Stage-1 Stage-0 is a root stage @@ -1307,6 +1442,41 @@ Stage: Stage-7 Conditional Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + c + TableScan + alias: c + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -1334,20 +1504,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - c - TableScan - alias: c Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1363,6 +1519,41 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + c + TableScan + alias: c + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -1390,20 +1581,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - c - TableScan - alias: c Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1419,6 +1596,41 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -1446,20 +1658,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - b - TableScan - alias: b Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) Index: ql/src/test/results/clientpositive/correlationoptimizer5.q.out =================================================================== --- ql/src/test/results/clientpositive/correlationoptimizer5.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/correlationoptimizer5.q.out (working copy) @@ -474,16 +474,39 @@ POSTHOOK: Lineage: dest_co2.key EXPRESSION [(t1)x.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: dest_co2.val SIMPLE [(t4)n.FieldSchema(name:val, type:string, comment:null), ] STAGE DEPENDENCIES: - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10, Stage-11 , consists of Stage-7, Stage-8, Stage-2 - Stage-7 has a backup stage: Stage-2 + Stage-14 is a root stage + Stage-10 depends on stages: Stage-14 + Stage-9 depends on stages: Stage-10, Stage-11 , consists of Stage-12, Stage-13, Stage-2 + Stage-12 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-12 Stage-0 depends on stages: Stage-2, Stage-7, Stage-8 Stage-3 depends on stages: Stage-0 - Stage-8 has a backup stage: Stage-2 + Stage-13 has a backup stage: Stage-2 + Stage-8 depends on stages: Stage-13 Stage-2 - Stage-11 is a root stage + Stage-15 is a root stage + Stage-11 depends on stages: Stage-15 STAGE PLANS: + Stage: Stage-14 + Map Reduce Local Work + Alias -> Map Local Tables: + d:m + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + d:m + TableScan + alias: m + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {val} + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-10 Map Reduce Map Operator Tree: @@ -513,19 +536,27 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - d:m - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - d:m - TableScan - alias: m - Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE Stage: Stage-9 Conditional Operator + Stage: Stage-12 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -552,13 +583,6 @@ name: default.dest_co3 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-0 Move Operator @@ -573,6 +597,23 @@ Stage: Stage-3 Stats-Aggr Operator + Stage: Stage-13 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Stage: Stage-8 Map Reduce Map Operator Tree: @@ -599,13 +640,6 @@ name: default.dest_co3 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-2 Map Reduce @@ -646,6 +680,25 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_co3 + Stage: Stage-15 + Map Reduce Local Work + Alias -> Map Local Tables: + b:y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b:y + TableScan + alias: y + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-11 Map Reduce Map Operator Tree: @@ -675,15 +728,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b:y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b:y - TableScan - alias: y - Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: INSERT OVERWRITE TABLE dest_co3 SELECT b.key, d.val Index: ql/src/test/results/clientpositive/auto_join1.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join1.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join1.q.out (working copy) @@ -14,11 +14,31 @@ INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage + Stage-5 is a root stage + Stage-4 depends on stages: Stage-5 Stage-0 depends on stages: Stage-4 Stage-2 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -50,15 +70,6 @@ name: default.dest_j1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Move Operator Index: ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out (working copy) @@ -580,9 +580,11 @@ STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-4, Stage-1 - Stage-3 has a backup stage: Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-1 Stage-0 is a root stage @@ -590,6 +592,73 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 114 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -617,62 +686,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 114 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -852,6 +865,121 @@ GatherStats: false MultiFileSpray: false + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=2008-04-08 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_big + name: default.bucket_big + Partition + base file name: ds=2008-04-09 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_big + name: default.bucket_big + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -879,110 +1007,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: ds=2008-04-08 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_big - name: default.bucket_big - Partition - base file name: ds=2008-04-09 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_big - name: default.bucket_big - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/subquery_in_having.q.out =================================================================== --- ql/src/test/results/clientpositive/subquery_in_having.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/subquery_in_having.q.out (working copy) @@ -820,8 +820,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-5 is a root stage - Stage-7 depends on stages: Stage-2, Stage-5 , consists of Stage-6, Stage-3 - Stage-6 has a backup stage: Stage-3 + Stage-7 depends on stages: Stage-2, Stage-5 , consists of Stage-9, Stage-3 + Stage-9 has a backup stage: Stage-3 + Stage-6 depends on stages: Stage-9 Stage-3 Stage-10 is a root stage Stage-2 depends on stages: Stage-10 @@ -879,6 +880,23 @@ Stage: Stage-7 Conditional Operator + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} + 1 + keys: + 0 _col2 (type: bigint) + 1 _col0 (type: bigint) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -906,13 +924,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-3 Map Reduce @@ -1059,8 +1070,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1, Stage-3 , consists of Stage-4, Stage-2 - Stage-4 has a backup stage: Stage-2 + Stage-5 depends on stages: Stage-1, Stage-3 , consists of Stage-6, Stage-2 + Stage-6 has a backup stage: Stage-2 + Stage-4 depends on stages: Stage-6 Stage-2 Stage-3 is a root stage Stage-0 is a root stage @@ -1102,6 +1114,23 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} + 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -1129,13 +1158,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-2 Map Reduce Index: ql/src/test/results/clientpositive/mapjoin_subquery.q.out =================================================================== --- ql/src/test/results/clientpositive/mapjoin_subquery.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/mapjoin_subquery.q.out (working copy) @@ -17,10 +17,44 @@ JOIN srcpart z ON (subq.key1 = z.key and z.ds='2008-04-08' and z.hr=11) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-5 is a root stage + Stage-7 is a root stage + Stage-5 depends on stages: Stage-7 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + subq:x + Fetch Operator + limit: -1 + z + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq:x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + z + TableScan + alias: z + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -66,22 +100,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq:x - Fetch Operator - limit: -1 - z - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq:x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - z - TableScan - alias: z - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -235,10 +253,44 @@ order by subq.key1, z.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-3 is a root stage + Stage-8 is a root stage + Stage-3 depends on stages: Stage-8 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + subq:x + Fetch Operator + limit: -1 + z + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq:x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + z + TableScan + alias: z + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -282,22 +334,6 @@ value expressions: _col0 (type: string), _col1 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq:x - Fetch Operator - limit: -1 - z - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq:x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - z - TableScan - alias: z - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 69 Data size: 7032 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/bucket_map_join_1.q.out =================================================================== --- ql/src/test/results/clientpositive/bucket_map_join_1.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucket_map_join_1.q.out (working copy) @@ -102,10 +102,32 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 0 Data size: 21 Basic stats: PARTIAL Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -138,16 +160,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 0 Data size: 21 Basic stats: PARTIAL Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketcontext_4.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketcontext_4.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketcontext_4.q.out (working copy) @@ -139,10 +139,132 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket4outof4.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -176,116 +298,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-08/srcsortbucket3outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-08/srcsortbucket4outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket4outof4.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/auto_join17.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join17.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join17.q.out (working copy) @@ -14,11 +14,31 @@ INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.* POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage + Stage-5 is a root stage + Stage-4 depends on stages: Stage-5 Stage-0 depends on stages: Stage-4 Stage-2 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -50,15 +70,6 @@ name: default.dest1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Move Operator Index: ql/src/test/results/clientpositive/auto_join26.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join26.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join26.q.out (working copy) @@ -14,11 +14,31 @@ SELECT x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -54,15 +74,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) Index: ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out =================================================================== --- ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out (working copy) @@ -186,10 +186,123 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 500 + partition_columns part + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 2 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 500 + partition_columns part + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -222,107 +335,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 500 - partition_columns part - partition_columns.types string - rawDataSize 5312 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 2 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 500 - partition_columns part - partition_columns.types string - rawDataSize 5312 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/mapjoin_test_outer.q.out =================================================================== --- ql/src/test/results/clientpositive/mapjoin_test_outer.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/mapjoin_test_outer.q.out (working copy) @@ -264,10 +264,48 @@ POSTHOOK: Lineage: dest_2.key SIMPLE [(dest_1)dest_1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: dest_2.value SIMPLE [(dest_1)dest_1.FieldSchema(name:value, type:string, comment:null), ] STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-4 is a root stage + Stage-1 depends on stages: Stage-4 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 + src2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + src2 + TableScan + alias: src2 + Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -299,22 +337,6 @@ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - src2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - src2 - TableScan - alias: src2 - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 19 Data size: 88 Basic stats: COMPLETE Column stats: NONE @@ -1102,10 +1124,48 @@ POSTHOOK: Lineage: dest_2.key SIMPLE [(dest_1)dest_1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: dest_2.value SIMPLE [(dest_1)dest_1.FieldSchema(name:value, type:string, comment:null), ] STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 + src2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + src2 + TableScan + alias: src2 + Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -1137,22 +1197,6 @@ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - src2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - src2 - TableScan - alias: src2 - Statistics: Num rows: 1 Data size: 13 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 19 Data size: 88 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/bucketmapjoin9.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin9.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin9.q.out (working copy) @@ -132,10 +132,78 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 3 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 3 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 4200 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 1050 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -168,62 +236,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 3 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 3 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 4200 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 1050 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -439,10 +451,78 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name value + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -475,62 +555,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name value - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketmapjoin13.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin13.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin13.q.out (working copy) @@ -136,10 +136,78 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 500 + partition_columns part + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -172,62 +240,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 500 - partition_columns part - partition_columns.types string - rawDataSize 5312 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -457,10 +469,85 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 500 + partition_columns part + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {part=2/000000_0=[part=1/000000_0], part=2/000001_0=[part=1/000001_0]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -494,69 +581,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 500 - partition_columns part - partition_columns.types string - rawDataSize 5312 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {part=2/000000_0=[part=1/000000_0], part=2/000001_0=[part=1/000001_0]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -743,10 +767,85 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 500 + partition_columns part + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {part=2/000000_0=[part=1/000000_0], part=2/000001_0=[part=1/000001_0]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -780,69 +879,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 500 - partition_columns part - partition_columns.types string - rawDataSize 5312 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {part=2/000000_0=[part=1/000000_0], part=2/000001_0=[part=1/000001_0]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1031,10 +1067,85 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 500 + partition_columns part + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {part=2/000000_0=[part=1/000000_0], part=2/000001_0=[part=1/000001_0]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -1068,69 +1179,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 500 - partition_columns part - partition_columns.types string - rawDataSize 5312 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {part=2/000000_0=[part=1/000000_0], part=2/000001_0=[part=1/000001_0]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/join28.q.out =================================================================== --- ql/src/test/results/clientpositive/join28.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join28.q.out (working copy) @@ -26,11 +26,45 @@ JOIN srcpart z ON (subq.key1 = z.key and z.ds='2008-04-08' and z.hr=11) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-6 is a root stage + Stage-8 is a root stage + Stage-6 depends on stages: Stage-8 Stage-0 depends on stages: Stage-6 Stage-3 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + subq:x + Fetch Operator + limit: -1 + z + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + subq:x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + z + TableScan + alias: z + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -77,22 +111,6 @@ name: default.dest_j1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - subq:x - Fetch Operator - limit: -1 - z - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - subq:x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - z - TableScan - alias: z - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Move Operator Index: ql/src/test/results/clientpositive/auto_join21.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join21.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join21.q.out (working copy) @@ -13,10 +13,30 @@ Stage: Stage-6 Map Reduce Local Work Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 src2 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 {(key < 10)} + 1 + 2 {(key < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) src2 TableScan alias: src2 @@ -26,13 +46,17 @@ Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: - 0 + 0 {key} {value} 1 {key} {value} 2 {key} {value} filter predicates: - 0 + 0 {(key < 10)} 1 2 {(key < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) Stage: Stage-2 Map Reduce @@ -69,17 +93,6 @@ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - src2 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - src2 Reduce Operator Tree: Extract Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/join37.q.out =================================================================== --- ql/src/test/results/clientpositive/join37.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join37.q.out (working copy) @@ -16,7 +16,8 @@ FROM src1 x JOIN src y ON (x.key = y.key) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -26,6 +27,25 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -57,15 +77,6 @@ name: default.dest_j1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE Stage: Stage-7 Conditional Operator Index: ql/src/test/results/clientpositive/bucketmapjoin4.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin4.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin4.q.out (working copy) @@ -169,7 +169,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6 Stage-5 Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 @@ -179,6 +180,34 @@ Stage-7 depends on stages: Stage-6 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -231,23 +260,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -689,7 +701,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -699,6 +712,34 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -756,23 +797,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/auto_join30.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join30.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join30.q.out (working copy) @@ -16,10 +16,12 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1, Stage-4 , consists of Stage-5, Stage-6, Stage-2 - Stage-5 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-1, Stage-4 , consists of Stage-8, Stage-9, Stage-2 + Stage-8 has a backup stage: Stage-2 + Stage-5 depends on stages: Stage-8 Stage-3 depends on stages: Stage-2, Stage-5, Stage-6 - Stage-6 has a backup stage: Stage-2 + Stage-9 has a backup stage: Stage-2 + Stage-6 depends on stages: Stage-9 Stage-2 Stage-4 is a root stage Stage-0 is a root stage @@ -53,6 +55,23 @@ Stage: Stage-7 Conditional Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -82,13 +101,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-3 Map Reduce @@ -116,6 +128,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -145,13 +174,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-2 Map Reduce @@ -260,8 +282,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-4 , consists of Stage-5, Stage-2 - Stage-5 has a backup stage: Stage-2 + Stage-6 depends on stages: Stage-1, Stage-4 , consists of Stage-7, Stage-2 + Stage-7 has a backup stage: Stage-2 + Stage-5 depends on stages: Stage-7 Stage-3 depends on stages: Stage-2, Stage-5 Stage-2 Stage-4 is a root stage @@ -296,6 +319,23 @@ Stage: Stage-6 Conditional Operator + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -325,13 +365,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-3 Map Reduce @@ -466,8 +499,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-4 , consists of Stage-5, Stage-2 - Stage-5 has a backup stage: Stage-2 + Stage-6 depends on stages: Stage-1, Stage-4 , consists of Stage-7, Stage-2 + Stage-7 has a backup stage: Stage-2 + Stage-5 depends on stages: Stage-7 Stage-3 depends on stages: Stage-2, Stage-5 Stage-2 Stage-4 is a root stage @@ -502,6 +536,23 @@ Stage: Stage-6 Conditional Operator + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -531,13 +582,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-3 Map Reduce @@ -678,11 +722,14 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-9 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-6, Stage-7, Stage-8, Stage-2 - Stage-6 has a backup stage: Stage-2 + Stage-9 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-10, Stage-11, Stage-12, Stage-2 + Stage-10 has a backup stage: Stage-2 + Stage-6 depends on stages: Stage-10 Stage-3 depends on stages: Stage-2, Stage-6, Stage-7, Stage-8 - Stage-7 has a backup stage: Stage-2 - Stage-8 has a backup stage: Stage-2 + Stage-11 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-11 + Stage-12 has a backup stage: Stage-2 + Stage-8 depends on stages: Stage-12 Stage-2 Stage-4 is a root stage Stage-5 is a root stage @@ -717,6 +764,39 @@ Stage: Stage-9 Conditional Operator + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + $INTNAME2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + $INTNAME2 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -749,18 +829,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - $INTNAME2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan - $INTNAME2 - TableScan Stage: Stage-3 Map Reduce @@ -788,6 +856,39 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-11 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -820,19 +921,40 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan - $INTNAME1 - TableScan + Stage: Stage-12 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + $INTNAME2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + $INTNAME2 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Stage: Stage-8 Map Reduce Map Operator Tree: @@ -865,18 +987,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - $INTNAME2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan - $INTNAME2 - TableScan Stage: Stage-2 Map Reduce @@ -1030,10 +1140,12 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-8 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-6, Stage-7, Stage-2 - Stage-6 has a backup stage: Stage-2 + Stage-8 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-9, Stage-10, Stage-2 + Stage-9 has a backup stage: Stage-2 + Stage-6 depends on stages: Stage-9 Stage-3 depends on stages: Stage-2, Stage-6, Stage-7 - Stage-7 has a backup stage: Stage-2 + Stage-10 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-10 Stage-2 Stage-4 is a root stage Stage-5 is a root stage @@ -1068,6 +1180,39 @@ Stage: Stage-8 Conditional Operator + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + $INTNAME2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + $INTNAME2 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -1100,18 +1245,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - $INTNAME2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan - $INTNAME2 - TableScan Stage: Stage-3 Map Reduce @@ -1139,6 +1272,39 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -1171,18 +1337,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan - $INTNAME1 - TableScan Stage: Stage-2 Map Reduce @@ -1336,8 +1490,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-6, Stage-2 - Stage-6 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-8, Stage-2 + Stage-8 has a backup stage: Stage-2 + Stage-6 depends on stages: Stage-8 Stage-3 depends on stages: Stage-2, Stage-6 Stage-2 Stage-4 is a root stage @@ -1373,6 +1528,39 @@ Stage: Stage-7 Conditional Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + $INTNAME2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + $INTNAME2 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -1405,18 +1593,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - $INTNAME2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan - $INTNAME2 - TableScan Stage: Stage-3 Map Reduce @@ -1596,8 +1772,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-6, Stage-2 - Stage-6 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-8, Stage-2 + Stage-8 has a backup stage: Stage-2 + Stage-6 depends on stages: Stage-8 Stage-3 depends on stages: Stage-2, Stage-6 Stage-2 Stage-4 is a root stage @@ -1633,6 +1810,39 @@ Stage: Stage-7 Conditional Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + $INTNAME2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + $INTNAME2 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -1665,18 +1875,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - $INTNAME2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan - $INTNAME2 - TableScan Stage: Stage-3 Map Reduce @@ -1856,8 +2054,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-6, Stage-2 - Stage-6 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-8, Stage-2 + Stage-8 has a backup stage: Stage-2 + Stage-6 depends on stages: Stage-8 Stage-3 depends on stages: Stage-2, Stage-6 Stage-2 Stage-4 is a root stage @@ -1893,6 +2092,39 @@ Stage: Stage-7 Conditional Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + $INTNAME2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + $INTNAME2 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {_col0} {_col1} + 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -1925,18 +2157,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - $INTNAME2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan - $INTNAME2 - TableScan Stage: Stage-3 Map Reduce Index: ql/src/test/results/clientpositive/smb_mapjoin_13.q.out =================================================================== --- ql/src/test/results/clientpositive/smb_mapjoin_13.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/smb_mapjoin_13.q.out (working copy) @@ -329,10 +329,32 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 UDFToDouble(key) (type: double) + 1 UDFToDouble(value) (type: double) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -364,16 +386,6 @@ value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/join32.q.out =================================================================== --- ql/src/test/results/clientpositive/join32.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join32.q.out (working copy) @@ -95,11 +95,95 @@ STAGE DEPENDENCIES: - Stage-5 is a root stage + Stage-7 is a root stage + Stage-5 depends on stages: Stage-7 Stage-0 depends on stages: Stage-5 Stage-2 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + z + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + z + TableScan + alias: z + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col5} {_col0} + 1 {value} + keys: + 0 _col1 (type: string) + 1 value (type: string) + Position of Big Table: 0 + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -163,70 +247,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - z - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 11 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 0 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 0 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - z - TableScan - alias: z - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/join_star.q.out =================================================================== --- ql/src/test/results/clientpositive/join_star.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join_star.q.out (working copy) @@ -123,10 +123,30 @@ POSTHOOK: query: explain select m1, m2, f2 from fact join dim1 on fact.d1=dim1.f1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-3 is a root stage + Stage-4 is a root stage + Stage-3 depends on stages: Stage-4 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + dim1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + dim1 + TableScan + alias: dim1 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {m1} {m2} + 1 {f2} + keys: + 0 d1 (type: int) + 1 f1 (type: int) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -157,15 +177,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - dim1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - dim1 - TableScan - alias: dim1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -196,10 +207,44 @@ POSTHOOK: query: explain select m1, m2, f2, f4 from fact join dim1 on fact.d1=dim1.f1 join dim2 on fact.d2=dim2.f3 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage + Stage-6 is a root stage + Stage-4 depends on stages: Stage-6 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + dim1 + Fetch Operator + limit: -1 + dim2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + dim1 + TableScan + alias: dim1 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {m1} {m2} {d2} + 1 {f2} + keys: + 0 d1 (type: int) + 1 f1 (type: int) + dim2 + TableScan + alias: dim2 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col7} {_col0} {_col1} + 1 {f4} + keys: + 0 _col3 (type: int) + 1 f3 (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -241,22 +286,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - dim1 - Fetch Operator - limit: -1 - dim2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - dim1 - TableScan - alias: dim1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - dim2 - TableScan - alias: dim2 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -289,10 +318,44 @@ POSTHOOK: query: explain select m1, m2, f2, f4 from fact join dim1 on fact.d1= dim1.f1 join dim2 on dim1.f2 = dim2.f3 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage + Stage-6 is a root stage + Stage-4 depends on stages: Stage-6 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + dim1 + Fetch Operator + limit: -1 + dim2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + dim1 + TableScan + alias: dim1 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {m1} {m2} + 1 {f2} + keys: + 0 d1 (type: int) + 1 f1 (type: int) + dim2 + TableScan + alias: dim2 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col7} {_col0} {_col1} + 1 {f4} + keys: + 0 _col7 (type: int) + 1 f3 (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -334,22 +397,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - dim1 - Fetch Operator - limit: -1 - dim2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - dim1 - TableScan - alias: dim1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - dim2 - TableScan - alias: dim2 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -382,10 +429,44 @@ POSTHOOK: query: explain select m1, m2, f2, f4 from fact Left outer join dim1 on fact.d1= dim1.f1 Left outer join dim2 on dim1.f2 = dim2.f3 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage + Stage-6 is a root stage + Stage-4 depends on stages: Stage-6 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + dim1 + Fetch Operator + limit: -1 + dim2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + dim1 + TableScan + alias: dim1 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {m1} {m2} + 1 {f2} + keys: + 0 d1 (type: int) + 1 f1 (type: int) + dim2 + TableScan + alias: dim2 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col7} {_col0} {_col1} + 1 {f4} + keys: + 0 _col7 (type: int) + 1 f3 (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -427,22 +508,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - dim1 - Fetch Operator - limit: -1 - dim2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - dim1 - TableScan - alias: dim1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - dim2 - TableScan - alias: dim2 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -491,10 +556,118 @@ Left outer Join dim7 on dim6.f12 = dim7.f13 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-12 is a root stage + Stage-18 is a root stage + Stage-12 depends on stages: Stage-18 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-18 + Map Reduce Local Work + Alias -> Map Local Tables: + dim1 + Fetch Operator + limit: -1 + dim2 + Fetch Operator + limit: -1 + dim3 + Fetch Operator + limit: -1 + dim4 + Fetch Operator + limit: -1 + dim5 + Fetch Operator + limit: -1 + dim6 + Fetch Operator + limit: -1 + dim7 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + dim1 + TableScan + alias: dim1 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {m1} {m2} {d2} + 1 {f2} + keys: + 0 d1 (type: int) + 1 f1 (type: int) + dim2 + TableScan + alias: dim2 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col7} {_col0} {_col1} {_col3} + 1 {f4} + keys: + 0 _col7 (type: int) + 1 f3 (type: int) + dim3 + TableScan + alias: dim3 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col11} {_col1} {_col4} {_col5} + 1 {f6} + keys: + 0 _col7 (type: int) + 1 f5 (type: int) + dim4 + TableScan + alias: dim4 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col15} {_col1} {_col5} {_col8} {_col9} + 1 {f8} + 2 {f12} + keys: + 0 _col15 (type: int) + 1 f7 (type: int) + 2 f11 (type: int) + dim5 + TableScan + alias: dim5 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col1} {_col5} {_col9} {_col12} {_col13} {_col23} {_col19} + 1 {f10} + keys: + 0 _col19 (type: int) + 1 f9 (type: int) + dim6 + TableScan + alias: dim6 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col15} {_col1} {_col5} {_col8} {_col9} + 1 {f8} + 2 {f12} + keys: + 0 _col15 (type: int) + 1 f7 (type: int) + 2 f11 (type: int) + dim7 + TableScan + alias: dim7 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col1} {_col5} {_col9} {_col12} {_col13} {_col19} {_col27} {_col23} + 1 {f14} + keys: + 0 _col19 (type: int) + 1 f13 (type: int) + Stage: Stage-12 Map Reduce Map Operator Tree: @@ -583,57 +756,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - dim1 - Fetch Operator - limit: -1 - dim2 - Fetch Operator - limit: -1 - dim3 - Fetch Operator - limit: -1 - dim4 - Fetch Operator - limit: -1 - dim5 - Fetch Operator - limit: -1 - dim6 - Fetch Operator - limit: -1 - dim7 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - dim1 - TableScan - alias: dim1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - dim2 - TableScan - alias: dim2 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - dim3 - TableScan - alias: dim3 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - dim4 - TableScan - alias: dim4 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - dim5 - TableScan - alias: dim5 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - dim6 - TableScan - alias: dim6 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE - dim7 - TableScan - alias: dim7 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator Index: ql/src/test/results/clientpositive/multiMapJoin2.q.out =================================================================== --- ql/src/test/results/clientpositive/multiMapJoin2.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/multiMapJoin2.q.out (working copy) @@ -15,10 +15,44 @@ ORDER BY tmp.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-8 is a root stage + Stage-2 depends on stages: Stage-8 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + null-subquery1:tmp-subquery1:y1 + Fetch Operator + limit: -1 + null-subquery2:tmp-subquery2:y2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery1:tmp-subquery1:y1 + TableScan + alias: y1 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + null-subquery2:tmp-subquery2:y2 + TableScan + alias: y2 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -82,22 +116,6 @@ value expressions: _col0 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - null-subquery1:tmp-subquery1:y1 - Fetch Operator - limit: -1 - null-subquery2:tmp-subquery2:y2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - null-subquery1:tmp-subquery1:y1 - TableScan - alias: y1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - null-subquery2:tmp-subquery2:y2 - TableScan - alias: y2 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE @@ -131,6 +149,7 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-8:MAPREDLOCAL RUN: Stage-2:MAPRED 128 128 @@ -235,11 +254,32 @@ ORDER BY tmp.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-2 depends on stages: Stage-7 + Stage-9 is a root stage + Stage-7 depends on stages: Stage-9 + Stage-8 depends on stages: Stage-7 + Stage-2 depends on stages: Stage-8 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + null-subquery1:tmp-subquery1:y1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery1:tmp-subquery1:y1 + TableScan + alias: y1 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -269,16 +309,26 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - null-subquery1:tmp-subquery1:y1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - null-subquery1:tmp-subquery1:y1 - TableScan - alias: y1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + null-subquery2:tmp-subquery2:y2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery2:tmp-subquery2:y2 + TableScan + alias: y2 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -325,15 +375,6 @@ value expressions: _col0 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - null-subquery2:tmp-subquery2:y2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - null-subquery2:tmp-subquery2:y2 - TableScan - alias: y2 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE @@ -367,7 +408,9 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-9:MAPREDLOCAL RUN: Stage-7:MAPRED +RUN: Stage-8:MAPREDLOCAL RUN: Stage-2:MAPRED 128 128 @@ -467,7 +510,8 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-4 is a root stage - Stage-2 depends on stages: Stage-4 + Stage-6 depends on stages: Stage-4 + Stage-2 depends on stages: Stage-6 Stage-0 is a root stage STAGE PLANS: @@ -508,6 +552,25 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + null-subquery2:tmp-subquery2:y2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery2:tmp-subquery2:y2 + TableScan + alias: y2 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -554,15 +617,6 @@ value expressions: _col0 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - null-subquery2:tmp-subquery2:y2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - null-subquery2:tmp-subquery2:y2 - TableScan - alias: y2 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 64 Data size: 6501 Basic stats: COMPLETE Column stats: NONE @@ -597,6 +651,7 @@ POSTHOOK: Input: default@src1 #### A masked pattern was here #### RUN: Stage-4:MAPRED +RUN: Stage-6:MAPREDLOCAL RUN: Stage-2:MAPRED 128 @@ -690,17 +745,40 @@ ORDER BY key, cnt POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-12 depends on stages: Stage-2, Stage-8 , consists of Stage-10, Stage-11, Stage-3 - Stage-10 has a backup stage: Stage-3 + Stage-17 is a root stage + Stage-2 depends on stages: Stage-17 + Stage-12 depends on stages: Stage-2, Stage-8 , consists of Stage-15, Stage-16, Stage-3 + Stage-15 has a backup stage: Stage-3 + Stage-10 depends on stages: Stage-15 Stage-4 depends on stages: Stage-3, Stage-10, Stage-11 Stage-5 depends on stages: Stage-4 - Stage-11 has a backup stage: Stage-3 + Stage-16 has a backup stage: Stage-3 + Stage-11 depends on stages: Stage-16 Stage-3 - Stage-8 is a root stage + Stage-18 is a root stage + Stage-8 depends on stages: Stage-18 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-17 + Map Reduce Local Work + Alias -> Map Local Tables: + tmp2:y2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + tmp2:y2 + TableScan + alias: y2 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -734,15 +812,6 @@ Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - tmp2:y2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - tmp2:y2 - TableScan - alias: y2 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) @@ -763,6 +832,23 @@ Stage: Stage-12 Conditional Operator + Stage: Stage-15 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-10 Map Reduce Map Operator Tree: @@ -793,13 +879,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-4 Map Reduce @@ -849,6 +928,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-16 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-11 Map Reduce Map Operator Tree: @@ -879,13 +975,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-3 Map Reduce @@ -929,6 +1018,25 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-18 + Map Reduce Local Work + Alias -> Map Local Tables: + tmp1:y1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + tmp1:y1 + TableScan + alias: y1 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-8 Map Reduce Map Operator Tree: @@ -962,15 +1070,6 @@ Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - tmp1:y1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - tmp1:y1 - TableScan - alias: y1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) @@ -1020,9 +1119,12 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-17:MAPREDLOCAL +RUN: Stage-18:MAPREDLOCAL RUN: Stage-2:MAPRED RUN: Stage-8:MAPRED RUN: Stage-12:CONDITIONAL +RUN: Stage-16:MAPREDLOCAL RUN: Stage-11:MAPRED RUN: Stage-4:MAPRED RUN: Stage-5:MAPRED @@ -1074,11 +1176,45 @@ ORDER BY key, cnt POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-9 is a root stage + Stage-2 depends on stages: Stage-9 Stage-3 depends on stages: Stage-2 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + tmp1:y1 + Fetch Operator + limit: -1 + tmp2:y2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + tmp1:y1 + TableScan + alias: y1 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + tmp2:y2 + TableScan + alias: y2 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -1140,22 +1276,6 @@ Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - tmp1:y1 - Fetch Operator - limit: -1 - tmp2:y2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - tmp1:y1 - TableScan - alias: y1 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - tmp2:y2 - TableScan - alias: y2 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator Statistics: Num rows: 126 Data size: 12786 Basic stats: COMPLETE Column stats: NONE @@ -1294,6 +1414,7 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-9:MAPREDLOCAL RUN: Stage-2:MAPRED RUN: Stage-3:MAPRED 128 1 @@ -1351,13 +1472,16 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-7 is a root stage - Stage-10 depends on stages: Stage-2, Stage-7 , consists of Stage-8, Stage-9, Stage-3 - Stage-8 has a backup stage: Stage-3 + Stage-10 depends on stages: Stage-2, Stage-7 , consists of Stage-12, Stage-13, Stage-3 + Stage-12 has a backup stage: Stage-3 + Stage-8 depends on stages: Stage-12 Stage-4 depends on stages: Stage-3, Stage-8, Stage-9 Stage-5 depends on stages: Stage-4 - Stage-9 has a backup stage: Stage-3 + Stage-13 has a backup stage: Stage-3 + Stage-9 depends on stages: Stage-13 Stage-3 - Stage-2 is a root stage + Stage-14 is a root stage + Stage-2 depends on stages: Stage-14 Stage-0 is a root stage STAGE PLANS: @@ -1401,6 +1525,23 @@ Stage: Stage-10 Conditional Operator + Stage: Stage-12 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-8 Map Reduce Map Operator Tree: @@ -1431,13 +1572,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-4 Map Reduce @@ -1487,6 +1621,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-13 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-9 Map Reduce Map Operator Tree: @@ -1517,13 +1668,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-3 Map Reduce @@ -1567,6 +1711,25 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-14 + Map Reduce Local Work + Alias -> Map Local Tables: + tmp2:y2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + tmp2:y2 + TableScan + alias: y2 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -1600,15 +1763,6 @@ Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - tmp2:y2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - tmp2:y2 - TableScan - alias: y2 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) @@ -1659,8 +1813,10 @@ POSTHOOK: Input: default@src1 #### A masked pattern was here #### RUN: Stage-7:MAPRED +RUN: Stage-14:MAPREDLOCAL RUN: Stage-2:MAPRED RUN: Stage-10:CONDITIONAL +RUN: Stage-12:MAPREDLOCAL RUN: Stage-8:MAPRED RUN: Stage-4:MAPRED RUN: Stage-5:MAPRED @@ -1712,11 +1868,31 @@ ORDER BY key, cnt POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-7 is a root stage + Stage-2 depends on stages: Stage-7 Stage-3 depends on stages: Stage-2 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + tmp2:y2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + tmp2:y2 + TableScan + alias: y2 + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -1767,15 +1943,6 @@ Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - tmp2:y2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - tmp2:y2 - TableScan - alias: y2 - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator Statistics: Num rows: 65 Data size: 6609 Basic stats: COMPLETE Column stats: NONE @@ -1914,6 +2081,7 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### +RUN: Stage-7:MAPREDLOCAL RUN: Stage-2:MAPRED RUN: Stage-3:MAPRED 128 1 @@ -1986,10 +2154,30 @@ POSTHOOK: Lineage: part_table PARTITION(partitionid=2).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-5 is a root stage + Stage-2 depends on stages: Stage-5 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + y + TableScan + alias: y + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -2019,15 +2207,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - y - TableScan - alias: y - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -2070,6 +2249,7 @@ POSTHOOK: Lineage: part_table PARTITION(partitionid=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-5:MAPREDLOCAL RUN: Stage-2:MAPRED 121 PREHOOK: query: -- HIVE-5891 Alias conflict when merging multiple mapjoin tasks into their common @@ -2103,12 +2283,34 @@ POSTHOOK: Lineage: part_table PARTITION(partitionid=2).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-4 depends on stages: Stage-2, Stage-7 - Stage-7 is a root stage + Stage-17 is a root stage + Stage-2 depends on stages: Stage-17 + Stage-16 depends on stages: Stage-2, Stage-7 + Stage-4 depends on stages: Stage-16 + Stage-18 is a root stage + Stage-7 depends on stages: Stage-18 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-17 + Map Reduce Local Work + Alias -> Map Local Tables: + null-subquery2:x-subquery2:tmp:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery2:x-subquery2:tmp:a + TableScan + alias: a + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -2142,15 +2344,6 @@ Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - null-subquery2:x-subquery2:tmp:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - null-subquery2:x-subquery2:tmp:a - TableScan - alias: a - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) @@ -2168,6 +2361,39 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-16 + Map Reduce Local Work + Alias -> Map Local Tables: + null-subquery1:x-subquery1:c + Fetch Operator + limit: -1 + null-subquery2:x-subquery2:c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery1:x-subquery1:c + TableScan + alias: c + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {key} + keys: + 0 _col0 (type: string) + 1 key (type: string) + null-subquery2:x-subquery2:c + TableScan + alias: c + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {key} + keys: + 0 _col0 (type: string) + 1 key (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -2231,23 +2457,26 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - null-subquery1:x-subquery1:c - Fetch Operator - limit: -1 - null-subquery2:x-subquery2:c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - null-subquery1:x-subquery1:c - TableScan - alias: c - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - null-subquery2:x-subquery2:c - TableScan - alias: c - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Stage: Stage-18 + Map Reduce Local Work + Alias -> Map Local Tables: + null-subquery1:x-subquery1:tmp:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery1:x-subquery1:tmp:a + TableScan + alias: a + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -2281,15 +2510,6 @@ Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - null-subquery1:x-subquery1:tmp:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - null-subquery1:x-subquery1:tmp:a - TableScan - alias: a - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) @@ -2339,8 +2559,11 @@ POSTHOOK: Lineage: part_table PARTITION(partitionid=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: part_table PARTITION(partitionid=2).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-17:MAPREDLOCAL +RUN: Stage-18:MAPREDLOCAL RUN: Stage-2:MAPRED RUN: Stage-7:MAPRED +RUN: Stage-16:MAPREDLOCAL RUN: Stage-4:MAPRED 0 0 Index: ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_8.q.out (working copy) @@ -599,9 +599,11 @@ STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-4, Stage-1 - Stage-3 has a backup stage: Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-1 Stage-0 is a root stage @@ -609,6 +611,119 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 114 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 114 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -636,108 +751,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 114 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 114 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -964,6 +977,121 @@ GatherStats: false MultiFileSpray: false + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=2008-04-08 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_big + name: default.bucket_big + Partition + base file name: ds=2008-04-09 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_big + name: default.bucket_big + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -991,110 +1119,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: ds=2008-04-08 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_big - name: default.bucket_big - Partition - base file name: ds=2008-04-09 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_big - name: default.bucket_big - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1492,18 +1516,6 @@ PREHOOK: Input: default@bucket_small@ds=2008-04-08 PREHOOK: Input: default@bucket_small@ds=2008-04-09 #### A masked pattern was here #### -Execution failed with exit status: 2 -Obtaining error information - -Task failed! -Task ID: - Stage-3 - -Logs: - -#### A masked pattern was here #### -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask -ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask POSTHOOK: query: select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucket_big Index: ql/src/test/results/clientpositive/correlationoptimizer7.q.out =================================================================== --- ql/src/test/results/clientpositive/correlationoptimizer7.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/correlationoptimizer7.q.out (working copy) @@ -15,11 +15,32 @@ ON xx.key=yy.key ORDER BY xx.key, xx.cnt, yy.key, yy.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-2 depends on stages: Stage-4 + Stage-9 is a root stage + Stage-4 depends on stages: Stage-9 + Stage-8 depends on stages: Stage-4 + Stage-2 depends on stages: Stage-8 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + xx:y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + xx:y + TableScan + alias: y + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -55,15 +76,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - xx:y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - xx:y - TableScan - alias: y - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -82,6 +94,25 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + yy + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + yy + TableScan + alias: yy + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {key} {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -108,15 +139,6 @@ value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - yy - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - yy - TableScan - alias: yy - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE @@ -184,11 +206,31 @@ ON xx.key=yy.key ORDER BY xx.key, xx.cnt, yy.key, yy.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-6 is a root stage + Stage-1 depends on stages: Stage-6 Stage-2 depends on stages: Stage-1 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + xx:y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + xx:y + TableScan + alias: y + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -233,15 +275,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - xx:y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - xx:y - TableScan - alias: y - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator Statistics: Num rows: 64 Data size: 6609 Basic stats: COMPLETE Column stats: NONE @@ -384,11 +417,32 @@ ON xx.key=yy.key ORDER BY xx.key, xx.cnt, yy.key, yy.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-2 depends on stages: Stage-4 + Stage-9 is a root stage + Stage-4 depends on stages: Stage-9 + Stage-8 depends on stages: Stage-4 + Stage-2 depends on stages: Stage-8 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + xx:y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + xx:y + TableScan + alias: y + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -424,15 +478,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - xx:y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - xx:y - TableScan - alias: y - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -451,6 +496,25 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + yy + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + yy + TableScan + alias: yy + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {key} {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -477,15 +541,6 @@ value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - yy - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - yy - TableScan - alias: yy - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 34 Data size: 3459 Basic stats: COMPLETE Column stats: NONE @@ -553,11 +608,31 @@ ON xx.key=yy.key ORDER BY xx.key, xx.cnt, yy.key, yy.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-6 is a root stage + Stage-1 depends on stages: Stage-6 Stage-2 depends on stages: Stage-1 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + xx:y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + xx:y + TableScan + alias: y + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -602,15 +677,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - xx:y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - xx:y - TableScan - alias: y - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator Statistics: Num rows: 64 Data size: 6609 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/auto_join3.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join3.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join3.q.out (working copy) @@ -14,11 +14,49 @@ INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-5 is a root stage + Stage-6 is a root stage + Stage-5 depends on stages: Stage-6 Stage-0 depends on stages: Stage-5 Stage-2 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 + src2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + src2 + TableScan + alias: src2 + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 {value} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -53,22 +91,6 @@ name: default.dest1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - src2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - src2 - TableScan - alias: src2 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Move Operator Index: ql/src/test/results/clientpositive/explain_rearrange.q.out =================================================================== --- ql/src/test/results/clientpositive/explain_rearrange.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/explain_rearrange.q.out (working copy) @@ -58,10 +58,12 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage [MAPRED] - Stage-9 depends on stages: Stage-1, Stage-5 , consists of Stage-7, Stage-8, Stage-3 [CONDITIONAL] - Stage-7 has a backup stage: Stage-3 [MAPRED] + Stage-9 depends on stages: Stage-1, Stage-5 , consists of Stage-10, Stage-11, Stage-3 [CONDITIONAL] + Stage-10 has a backup stage: Stage-3 [MAPREDLOCAL] + Stage-7 depends on stages: Stage-10 [MAPRED] Stage-4 depends on stages: Stage-3, Stage-7, Stage-8 [MAPRED] - Stage-8 has a backup stage: Stage-3 [MAPRED] + Stage-11 has a backup stage: Stage-3 [MAPREDLOCAL] + Stage-8 depends on stages: Stage-11 [MAPRED] Stage-3 [MAPRED] Stage-5 is a root stage [MAPRED] Stage-0 is a root stage [FETCH] @@ -115,6 +117,23 @@ Stage: Stage-9 Conditional Operator + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -140,13 +159,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-4 Map Reduce @@ -165,6 +177,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-11 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + Stage: Stage-8 Map Reduce Map Operator Tree: @@ -190,13 +219,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-3 Map Reduce @@ -322,13 +344,15 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage [MAPRED] - Stage-2 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-5, Stage-6 [CONDITIONAL] - Stage-3 has a backup stage: Stage-6 [MAPRED] - Stage-4 depends on stages: Stage-6, Stage-3, Stage-5 [MAPRED] - Stage-5 has a backup stage: Stage-6 [MAPRED] - Stage-6 [MAPRED] - Stage-7 is a root stage [MAPRED] - Stage-8 is a root stage [FETCH] + Stage-2 depends on stages: Stage-1, Stage-9 , consists of Stage-3, Stage-6, Stage-8 [CONDITIONAL] + Stage-3 has a backup stage: Stage-8 [MAPREDLOCAL] + Stage-4 depends on stages: Stage-3 [MAPRED] + Stage-5 depends on stages: Stage-8, Stage-4, Stage-7 [MAPRED] + Stage-6 has a backup stage: Stage-8 [MAPREDLOCAL] + Stage-7 depends on stages: Stage-6 [MAPRED] + Stage-8 [MAPRED] + Stage-9 is a root stage [MAPRED] + Stage-10 is a root stage [FETCH] STAGE PLANS: Stage: Stage-1 @@ -380,6 +404,23 @@ Conditional Operator Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan @@ -404,15 +445,8 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan - Stage: Stage-4 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -429,7 +463,24 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-5 + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + + Stage: Stage-7 Map Reduce Map Operator Tree: TableScan @@ -454,15 +505,8 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan - Stage: Stage-6 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan @@ -495,7 +539,7 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-7 + Stage: Stage-9 Map Reduce Map Operator Tree: TableScan @@ -540,7 +584,7 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-8 + Stage: Stage-10 Fetch Operator limit: -1 @@ -587,12 +631,14 @@ STAGE DEPENDENCIES: Stage-1 is a root stage [MAPRED] Stage-2 is a root stage [MAPRED] - Stage-3 depends on stages: Stage-1, Stage-2 , consists of Stage-4, Stage-5, Stage-6 [CONDITIONAL] - Stage-4 has a backup stage: Stage-6 [MAPRED] - Stage-5 has a backup stage: Stage-6 [MAPRED] - Stage-6 [MAPRED] - Stage-7 depends on stages: Stage-6, Stage-4, Stage-5 [MAPRED] - Stage-8 is a root stage [FETCH] + Stage-3 depends on stages: Stage-1, Stage-2 , consists of Stage-4, Stage-6, Stage-8 [CONDITIONAL] + Stage-4 has a backup stage: Stage-8 [MAPREDLOCAL] + Stage-5 depends on stages: Stage-4 [MAPRED] + Stage-6 has a backup stage: Stage-8 [MAPREDLOCAL] + Stage-7 depends on stages: Stage-6 [MAPRED] + Stage-8 [MAPRED] + Stage-9 depends on stages: Stage-8, Stage-5, Stage-7 [MAPRED] + Stage-10 is a root stage [FETCH] STAGE PLANS: Stage: Stage-1 @@ -689,6 +735,23 @@ Conditional Operator Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -713,15 +776,25 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan - Stage: Stage-5 + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + + Stage: Stage-7 Map Reduce Map Operator Tree: TableScan @@ -746,15 +819,8 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan - Stage: Stage-6 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan @@ -787,7 +853,7 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-7 + Stage: Stage-9 Map Reduce Map Operator Tree: TableScan @@ -804,7 +870,7 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-8 + Stage: Stage-10 Fetch Operator limit: -1 @@ -852,11 +918,13 @@ Stage-1 is a root stage [MAPRED] Stage-2 is a root stage [MAPRED] Stage-3 depends on stages: Stage-1, Stage-2 , consists of Stage-4, Stage-5, Stage-6 [CONDITIONAL] - Stage-4 has a backup stage: Stage-6 [MAPRED] - Stage-5 has a backup stage: Stage-6 [MAPRED] + Stage-4 has a backup stage: Stage-6 [MAPREDLOCAL] + Stage-5 has a backup stage: Stage-6 [MAPREDLOCAL] Stage-6 [MAPRED] - Stage-7 depends on stages: Stage-6, Stage-4, Stage-5 [MAPRED] - Stage-8 is a root stage [FETCH] + Stage-7 depends on stages: Stage-4 [MAPRED] + Stage-8 depends on stages: Stage-5 [MAPRED] + Stage-9 depends on stages: Stage-6, Stage-7, Stage-8 [MAPRED] + Stage-10 is a root stage [FETCH] STAGE PLANS: Stage: Stage-1 @@ -953,70 +1021,38 @@ Conditional Operator Stage: Stage-4 - Map Reduce - Map Operator Tree: + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 TableScan - Map Join Operator - condition map: - Inner Join 0 to 1 + HashTable Sink Operator condition expressions: 0 {_col0} {_col1} 1 {_col1} keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Local Work: - Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-5 - Map Reduce - Map Operator Tree: + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME TableScan - Map Join Operator - condition map: - Inner Join 0 to 1 + HashTable Sink Operator condition expressions: 0 {_col0} {_col1} 1 {_col1} keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Select Operator - expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Local Work: - Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-6 Map Reduce @@ -1055,6 +1091,58 @@ Map Reduce Map Operator Tree: TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-8 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} {_col1} + 1 {_col1} + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Select Operator + expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-9 + Map Reduce + Map Operator Tree: + TableScan Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: bigint) sort order: +++ @@ -1068,7 +1156,7 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-8 + Stage: Stage-10 Fetch Operator limit: -1 Index: ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out (working copy) @@ -462,9 +462,11 @@ STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-4, Stage-1 - Stage-3 has a backup stage: Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-1 Stage-0 is a root stage @@ -472,6 +474,119 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 114 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 114 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -499,108 +614,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 114 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 114 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -778,6 +791,74 @@ GatherStats: false MultiFileSpray: false + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=2008-04-08 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_big + name: default.bucket_big + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -805,63 +886,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: ds=2008-04-08 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_big - name: default.bucket_big - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out (working copy) @@ -191,10 +191,39 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + r + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + r + TableScan + alias: r + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + r {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt], srcbucket22.txt=[srcbucket22.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -242,23 +271,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - r - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - r - TableScan - alias: r - Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - r {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt], srcbucket22.txt=[srcbucket22.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -368,10 +380,39 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + r + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + r + TableScan + alias: r + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + r {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt], srcbucket22.txt=[srcbucket22.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -419,23 +460,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - r - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - r - TableScan - alias: r - Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - r {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt], srcbucket22.txt=[srcbucket22.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -542,10 +566,32 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + r + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + r + TableScan + alias: r + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 (key + key) (type: double) + 1 UDFToDouble(key) (type: double) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -592,16 +638,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - r - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - r - TableScan - alias: r - Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -711,10 +747,32 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + r + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + r + TableScan + alias: r + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -761,16 +819,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - r - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - r - TableScan - alias: r - Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -880,10 +928,32 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + r + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + r + TableScan + alias: r + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -930,16 +1000,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - r - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - r - TableScan - alias: r - Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1049,10 +1109,32 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + r + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + r + TableScan + alias: r + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -1099,16 +1181,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - r - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - r - TableScan - alias: r - Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1218,10 +1290,32 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + r + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + r + TableScan + alias: r + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -1268,16 +1362,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - r - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - r - TableScan - alias: r - Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1387,10 +1471,32 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + r + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + r + TableScan + alias: r + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -1437,16 +1543,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - r - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - r - TableScan - alias: r - Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1556,10 +1652,32 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + r + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + r + TableScan + alias: r + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -1606,16 +1724,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - r - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - r - TableScan - alias: r - Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/skewjoin_noskew.q.out =================================================================== --- ql/src/test/results/clientpositive/skewjoin_noskew.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/skewjoin_noskew.q.out (working copy) @@ -6,8 +6,9 @@ POSTHOOK: type: CREATETABLE_AS_SELECT STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1 , consists of Stage-5, Stage-2 - Stage-5 + Stage-6 depends on stages: Stage-1 , consists of Stage-7, Stage-2 + Stage-7 + Stage-5 depends on stages: Stage-7 Stage-2 depends on stages: Stage-5 Stage-0 depends on stages: Stage-2 Stage-8 depends on stages: Stage-0 @@ -58,6 +59,23 @@ Stage: Stage-6 Conditional Operator + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + 1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + 1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {0_VALUE_0} {0_VALUE_1} + 1 + keys: + 0 joinkey0 (type: string) + 1 joinkey0 (type: string) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -83,13 +101,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - 1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - 1 - TableScan Stage: Stage-2 Map Reduce Index: ql/src/test/results/clientpositive/auto_sortmerge_join_15.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_15.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_15.q.out (working copy) @@ -45,8 +45,9 @@ POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-4 is a root stage , consists of Stage-3, Stage-1 - Stage-3 has a backup stage: Stage-1 + Stage-4 is a root stage , consists of Stage-5, Stage-1 + Stage-5 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-5 Stage-1 Stage-0 is a root stage @@ -54,6 +55,24 @@ Stage: Stage-4 Conditional Operator + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -78,14 +97,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -154,8 +165,9 @@ POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-4 is a root stage , consists of Stage-3, Stage-1 - Stage-3 has a backup stage: Stage-1 + Stage-4 is a root stage , consists of Stage-5, Stage-1 + Stage-5 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-5 Stage-1 Stage-0 is a root stage @@ -163,6 +175,24 @@ Stage: Stage-4 Conditional Operator + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -187,14 +217,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) Index: ql/src/test/results/clientpositive/bucketcontext_6.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketcontext_6.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketcontext_6.q.out (working copy) @@ -122,10 +122,39 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcsortbucket1outof4.txt=[srcsortbucket1outof4.txt, srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[srcsortbucket2outof4.txt, srcsortbucket4outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[srcsortbucket1outof4.txt, srcsortbucket3outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[srcsortbucket2outof4.txt, srcsortbucket4outof4.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -159,23 +188,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcsortbucket1outof4.txt=[srcsortbucket1outof4.txt, srcsortbucket3outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[srcsortbucket2outof4.txt, srcsortbucket4outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[srcsortbucket1outof4.txt, srcsortbucket3outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[srcsortbucket2outof4.txt, srcsortbucket4outof4.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/auto_join19.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join19.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join19.q.out (working copy) @@ -16,11 +16,31 @@ where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11') POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage + Stage-5 is a root stage + Stage-4 depends on stages: Stage-5 Stage-0 depends on stages: Stage-4 Stage-2 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + src2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src2 + TableScan + alias: src2 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -52,15 +72,6 @@ name: default.dest1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src2 - TableScan - alias: src2 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Move Operator Index: ql/src/test/results/clientpositive/auto_join28.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join28.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join28.q.out (working copy) @@ -13,10 +13,30 @@ Stage: Stage-6 Map Reduce Local Work Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 src2 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 {(key < 10)} + 1 + 2 {(key < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) src2 TableScan alias: src2 @@ -26,13 +46,17 @@ Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator condition expressions: - 0 + 0 {key} {value} 1 {key} {value} 2 {key} {value} filter predicates: - 0 + 0 {(key < 10)} 1 2 {(key < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) Stage: Stage-2 Map Reduce @@ -69,17 +93,6 @@ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - src2 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - src2 Reduce Operator Tree: Extract Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE @@ -339,6 +352,9 @@ src1 Fetch Operator limit: -1 + src2 + Fetch Operator + limit: -1 Alias -> Map Local Operator Tree: src1 TableScan @@ -350,12 +366,33 @@ HashTable Sink Operator condition expressions: 0 {key} {value} - 1 + 1 {key} {value} 2 {key} {value} filter predicates: 0 - 1 + 1 {(key > 10)} 2 {(key < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + src2 + TableScan + alias: src2 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} {value} + 2 {key} {value} + filter predicates: + 0 + 1 {(key > 10)} + 2 {(key < 10)} + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) Stage: Stage-2 Map Reduce @@ -392,17 +429,6 @@ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - src2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1 - src2 - TableScan - alias: src2 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Extract Statistics: Num rows: 63 Data size: 12786 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/bucketcontext_1.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketcontext_1.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketcontext_1.q.out (working copy) @@ -139,10 +139,86 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -176,70 +252,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/auto_join32.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join32.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join32.q.out (working copy) @@ -25,10 +25,30 @@ group by s.name POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-5 is a root stage + Stage-2 depends on stages: Stage-5 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + s + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + s + TableScan + alias: s + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {name} + 1 {registration} + keys: + 0 name (type: string) + 1 name (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -64,15 +84,6 @@ value expressions: _col2 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - s - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - s - TableScan - alias: s - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(DISTINCT KEY._col1:0._col0) Index: ql/src/test/results/clientpositive/bucketmapjoin10.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin10.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin10.q.out (working copy) @@ -190,10 +190,123 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 3 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 3 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 4200 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 3 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 2 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 3 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 1737 Data size: 6950 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -226,107 +339,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 3 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 3 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 4200 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 3 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 2 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 3 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 1737 Data size: 6950 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/join_reorder4.q.out =================================================================== --- ql/src/test/results/clientpositive/join_reorder4.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join_reorder4.q.out (working copy) @@ -48,10 +48,48 @@ POSTHOOK: query: explain select /*+ STREAMTABLE(a) */ a.*, b.*, c.* from T1 a join T2 b on a.key1=b.key2 join T3 c on a.key1=c.key3 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage + Stage-5 is a root stage + Stage-4 depends on stages: Stage-5 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key1} {val1} + 1 {key2} {val2} + 2 {key3} {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) + c + TableScan + alias: c + Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key1} {val1} + 1 {key2} {val2} + 2 {key3} {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -85,22 +123,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - c - TableScan - alias: c - Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE Stage: Stage-0 Fetch Operator @@ -124,10 +146,48 @@ POSTHOOK: query: explain select /*+ STREAMTABLE(b) */ a.*, b.*, c.* from T1 a join T2 b on a.key1=b.key2 join T3 c on a.key1=c.key3 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage + Stage-5 is a root stage + Stage-4 depends on stages: Stage-5 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key1} {val1} + 1 {key2} {val2} + 2 {key3} {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) + c + TableScan + alias: c + Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key1} {val1} + 1 {key2} {val2} + 2 {key3} {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -161,22 +221,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - c - TableScan - alias: c - Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE Stage: Stage-0 Fetch Operator @@ -200,10 +244,48 @@ POSTHOOK: query: explain select /*+ STREAMTABLE(c) */ a.*, b.*, c.* from T1 a join T2 b on a.key1=b.key2 join T3 c on a.key1=c.key3 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-4 is a root stage + Stage-5 is a root stage + Stage-4 depends on stages: Stage-5 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key1} {val1} + 1 {key2} {val2} + 2 {key3} {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) + c + TableScan + alias: c + Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key1} {val1} + 1 {key2} {val2} + 2 {key3} {val3} + keys: + 0 key1 (type: string) + 1 key2 (type: string) + 2 key3 (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -237,22 +319,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - c - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE - c - TableScan - alias: c - Statistics: Num rows: 0 Data size: 20 Basic stats: PARTIAL Column stats: NONE Stage: Stage-0 Fetch Operator Index: ql/src/test/results/clientpositive/join25.q.out =================================================================== --- ql/src/test/results/clientpositive/join25.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join25.q.out (working copy) @@ -16,7 +16,8 @@ FROM src1 x JOIN src y ON (x.key = y.key) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -26,6 +27,25 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -57,15 +77,6 @@ name: default.dest_j1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE Stage: Stage-7 Conditional Operator Index: ql/src/test/results/clientpositive/smb_mapjoin_15.q.out =================================================================== --- ql/src/test/results/clientpositive/smb_mapjoin_15.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/smb_mapjoin_15.q.out (working copy) @@ -813,10 +813,39 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 500 Data size: 7218 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {key2} {value} + 1 {key} {key2} {value} + keys: + 0 key (type: int), value (type: string) + 1 key (type: int), value (type: string) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {000000_0=[000000_0], 000001_0=[000001_0], 000002_0=[000002_0], 000003_0=[000003_0], 000004_0=[000004_0], 000005_0=[000005_0], 000006_0=[000006_0], 000007_0=[000007_0], 000008_0=[000008_0], 000009_0=[000009_0], 000010_0=[000010_0], 000011_0=[000011_0], 000012_0=[000012_0], 000013_0=[000013_0], 000014_0=[000014_0], 000015_0=[000015_0]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -849,23 +878,6 @@ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 500 Data size: 7218 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {000000_0=[000000_0], 000001_0=[000001_0], 000002_0=[000002_0], 000003_0=[000003_0], 000004_0=[000004_0], 000005_0=[000005_0], 000006_0=[000006_0], 000007_0=[000007_0], 000008_0=[000008_0], 000009_0=[000009_0], 000010_0=[000010_0], 000011_0=[000011_0], 000012_0=[000012_0], 000013_0=[000013_0], 000014_0=[000014_0], 000015_0=[000015_0]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/join34.q.out =================================================================== --- ql/src/test/results/clientpositive/join34.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join34.q.out (working copy) @@ -134,11 +134,33 @@ STAGE DEPENDENCIES: - Stage-5 is a root stage + Stage-6 is a root stage + Stage-5 depends on stages: Stage-6 Stage-0 depends on stages: Stage-5 Stage-2 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col1} + 1 {key} {value} + keys: + 0 _col0 (type: string) + 1 key (type: string) + Position of Big Table: 0 + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -256,16 +278,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketmapjoin1.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin1.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin1.q.out (working copy) @@ -213,29 +213,41 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {} + Alias Bucket File Name Mapping: + a {} + Stage: Stage-1 Map Reduce Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {} - Alias Bucket File Name Mapping: - a {} Stage: Stage-0 Fetch Operator @@ -412,7 +424,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6 Stage-5 Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 @@ -422,6 +435,80 @@ Stage-7 depends on stages: Stage-6 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part + partition_columns ds + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part + name: default.srcbucket_mapjoin_part + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {srcbucket20.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], srcbucket21.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -474,69 +561,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part - partition_columns ds - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part - name: default.srcbucket_mapjoin_part - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {srcbucket20.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], srcbucket21.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -993,7 +1017,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -1003,6 +1028,34 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -1060,23 +1113,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out (working copy) @@ -74,17 +74,37 @@ POSTHOOK: Lineage: test_table2 PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: test_table2 PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-6 is a root stage , consists of Stage-4, Stage-5, Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-0 depends on stages: Stage-1, Stage-4, Stage-5 Stage-3 depends on stages: Stage-0 - Stage-5 has a backup stage: Stage-1 + Stage-8 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-8 Stage-1 STAGE PLANS: Stage: Stage-6 Conditional Operator + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -110,14 +130,6 @@ value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b Reduce Operator Tree: Extract File Output Operator @@ -143,6 +155,24 @@ Stage: Stage-3 Stats-Aggr Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -168,14 +198,6 @@ value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a Reduce Operator Tree: Extract File Output Operator Index: ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out (working copy) @@ -431,9 +431,11 @@ STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-4, Stage-1 - Stage-3 has a backup stage: Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-1 Stage-0 is a root stage @@ -441,6 +443,26 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -468,15 +490,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -600,6 +613,26 @@ GatherStats: false MultiFileSpray: false + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -627,15 +660,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/correlationoptimizer4.q.out =================================================================== --- ql/src/test/results/clientpositive/correlationoptimizer4.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/correlationoptimizer4.q.out (working copy) @@ -359,11 +359,49 @@ GROUP BY y.key) tmp POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-7 is a root stage + Stage-2 depends on stages: Stage-7 Stage-3 depends on stages: Stage-2 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + tmp:x + Fetch Operator + limit: -1 + tmp:y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + tmp:x + TableScan + alias: x + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {key} + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + tmp:y + TableScan + alias: y + Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {key} + 2 + keys: + 0 key (type: int) + 1 key (type: int) + 2 key (type: int) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -402,22 +440,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - tmp:x - Fetch Operator - limit: -1 - tmp:y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - tmp:x - TableScan - alias: x - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE - tmp:y - TableScan - alias: y - Statistics: Num rows: 7 Data size: 30 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) Index: ql/src/test/results/clientpositive/skewjoin.q.out =================================================================== --- ql/src/test/results/clientpositive/skewjoin.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/skewjoin.q.out (working copy) @@ -75,8 +75,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-0 - Stage-4 + Stage-5 depends on stages: Stage-1 , consists of Stage-6, Stage-0 + Stage-6 + Stage-4 depends on stages: Stage-6 Stage-0 depends on stages: Stage-4 Stage-2 depends on stages: Stage-0 @@ -128,6 +129,23 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + 1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + 1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {0_VALUE_0} + 1 {1_VALUE_0} + keys: + 0 joinkey0 (type: string) + 1 joinkey0 (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -154,13 +172,6 @@ name: default.dest_j1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - 1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - 1 - TableScan Stage: Stage-0 Move Operator @@ -537,8 +548,9 @@ POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-2 - Stage-4 + Stage-5 depends on stages: Stage-1 , consists of Stage-6, Stage-2 + Stage-6 + Stage-4 depends on stages: Stage-6 Stage-2 depends on stages: Stage-4 Stage-0 is a root stage @@ -600,6 +612,23 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + 1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + 1 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {1_VALUE_0} {1_VALUE_1} + keys: + 0 joinkey0 (type: string) + 1 joinkey0 (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -629,13 +658,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - 1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - 1 - TableScan Stage: Stage-2 Map Reduce @@ -706,8 +728,9 @@ POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-2 - Stage-4 + Stage-5 depends on stages: Stage-1 , consists of Stage-6, Stage-2 + Stage-6 + Stage-4 depends on stages: Stage-6 Stage-2 depends on stages: Stage-4 Stage-0 is a root stage @@ -769,6 +792,23 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + 1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + 1 + TableScan + HashTable Sink Operator + condition expressions: + 0 + 1 {1_VALUE_0} {1_VALUE_1} + keys: + 0 joinkey0 (type: string), joinkey1 (type: double) + 1 joinkey0 (type: string), joinkey1 (type: double) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -798,13 +838,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - 1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - 1 - TableScan Stage: Stage-2 Map Reduce @@ -883,10 +916,12 @@ POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-5, Stage-6, Stage-2 - Stage-5 + Stage-7 depends on stages: Stage-1 , consists of Stage-8, Stage-9, Stage-2 + Stage-8 + Stage-5 depends on stages: Stage-8 Stage-2 depends on stages: Stage-5, Stage-6 - Stage-6 + Stage-9 + Stage-6 depends on stages: Stage-9 Stage-0 is a root stage STAGE PLANS: @@ -971,6 +1006,39 @@ Stage: Stage-7 Conditional Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + 1 + Fetch Operator + limit: -1 + 2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + 1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {0_VALUE_0} + 1 {1_VALUE_0} + 2 + keys: + 0 joinkey0 (type: string) + 1 joinkey0 (type: string) + 2 joinkey0 (type: string) + 2 + TableScan + HashTable Sink Operator + condition expressions: + 0 {0_VALUE_0} + 1 {1_VALUE_0} + 2 + keys: + 0 joinkey0 (type: string) + 1 joinkey0 (type: string) + 2 joinkey0 (type: string) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -1003,18 +1071,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - 1 - Fetch Operator - limit: -1 - 2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - 1 - TableScan - 2 - TableScan Stage: Stage-2 Map Reduce @@ -1042,6 +1098,39 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + 0 + Fetch Operator + limit: -1 + 2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + 0 + TableScan + HashTable Sink Operator + condition expressions: + 0 {0_VALUE_0} + 1 {1_VALUE_0} + 2 + keys: + 0 joinkey0 (type: string) + 1 joinkey0 (type: string) + 2 joinkey0 (type: string) + 2 + TableScan + HashTable Sink Operator + condition expressions: + 0 {0_VALUE_0} + 1 {1_VALUE_0} + 2 + keys: + 0 joinkey0 (type: string) + 1 joinkey0 (type: string) + 2 joinkey0 (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -1074,18 +1163,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - 0 - Fetch Operator - limit: -1 - 2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - 0 - TableScan - 2 - TableScan Stage: Stage-0 Fetch Operator @@ -1127,10 +1204,30 @@ POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + v + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + v + TableScan + alias: v + Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {val} + keys: + 0 (key + 1) (type: double) + 1 UDFToDouble(key) (type: double) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -1163,15 +1260,6 @@ value expressions: _col0 (type: bigint), _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - v - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - v - TableScan - alias: v - Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0), sum(VALUE._col1) Index: ql/src/test/results/clientpositive/mapjoin_distinct.q.out =================================================================== --- ql/src/test/results/clientpositive/mapjoin_distinct.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/mapjoin_distinct.q.out (working copy) @@ -11,11 +11,31 @@ SELECT /*+ MAPJOIN(d) */ DISTINCT c.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-4 is a root stage + Stage-1 depends on stages: Stage-4 Stage-3 depends on stages: Stage-1 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + d + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + d + TableScan + alias: d + Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -49,15 +69,6 @@ Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - d - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - d - TableScan - alias: d - Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) @@ -143,10 +154,30 @@ SELECT /*+ MAPJOIN(d) */ DISTINCT c.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + d + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + d + TableScan + alias: d + Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -180,15 +211,6 @@ Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - d - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - d - TableScan - alias: d - Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) @@ -252,11 +274,31 @@ SELECT /*+ MAPJOIN(d) */ DISTINCT c.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-4 is a root stage + Stage-1 depends on stages: Stage-4 Stage-3 depends on stages: Stage-1 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + d + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + d + TableScan + alias: d + Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -285,15 +327,6 @@ Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - d - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - d - TableScan - alias: d - Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) @@ -379,10 +412,30 @@ SELECT /*+ MAPJOIN(d) */ DISTINCT c.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + d + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + d + TableScan + alias: d + Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -411,15 +464,6 @@ Statistics: Num rows: 127 Data size: 12786 Basic stats: COMPLETE Column stats: NONE Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - d - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - d - TableScan - alias: d - Statistics: Num rows: 116 Data size: 11624 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) Index: ql/src/test/results/clientpositive/bucketcontext_8.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketcontext_8.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketcontext_8.q.out (working copy) @@ -156,10 +156,132 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -193,116 +315,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-09/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-09/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out (working copy) @@ -201,10 +201,212 @@ STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-8 is a root stage + Stage-2 depends on stages: Stage-8 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=2008-04-08 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 114 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + b + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=2008-04-08 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 3 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_medium + numFiles 3 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_medium { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 170 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 3 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_medium + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_medium { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_medium + name: default.bucket_medium + d + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=2008-04-08 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 3 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_medium + numFiles 3 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_medium { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 170 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 3 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_medium + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_medium { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_medium + name: default.bucket_medium + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 + b + TableScan + alias: b + Statistics: Num rows: 1 Data size: 170 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 + d + TableScan + alias: d + Statistics: Num rows: 0 Data size: 170 Basic stats: PARTIAL Column stats: COMPLETE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 + 1 + Position of Big Table: 0 + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -251,176 +453,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: ds=2008-04-08 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 114 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - b - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: ds=2008-04-08 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 3 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_medium - numFiles 3 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_medium { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 170 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 3 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_medium - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_medium { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_medium - name: default.bucket_medium - d - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: ds=2008-04-08 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 3 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_medium - numFiles 3 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_medium { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 170 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 3 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_medium - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_medium { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_medium - name: default.bucket_medium - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - b - TableScan - alias: b - Statistics: Num rows: 1 Data size: 170 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - d - TableScan - alias: d - Statistics: Num rows: 0 Data size: 170 Basic stats: PARTIAL Column stats: COMPLETE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketcontext_3.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketcontext_3.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketcontext_3.q.out (working copy) @@ -123,10 +123,132 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -160,116 +282,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 54 Data size: 5500 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcsortbucket1outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket2outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt], ds=2008-04-08/srcsortbucket3outof4.txt=[ds=2008-04-08/srcsortbucket1outof4.txt, ds=2008-04-09/srcsortbucket1outof4.txt], ds=2008-04-08/srcsortbucket4outof4.txt=[ds=2008-04-08/srcsortbucket2outof4.txt, ds=2008-04-09/srcsortbucket2outof4.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/subquery_multiinsert.q.out =================================================================== --- ql/src/test/results/clientpositive/subquery_multiinsert.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/subquery_multiinsert.q.out (working copy) @@ -535,8 +535,9 @@ POSTHOOK: Lineage: src_5.value EXPRESSION [(src)b.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: Stage-10 is a root stage - Stage-13 depends on stages: Stage-10, Stage-14 , consists of Stage-12, Stage-4 - Stage-12 has a backup stage: Stage-4 + Stage-13 depends on stages: Stage-10, Stage-14 , consists of Stage-16, Stage-4 + Stage-16 has a backup stage: Stage-4 + Stage-12 depends on stages: Stage-16 Stage-15 depends on stages: Stage-4, Stage-12 Stage-6 depends on stages: Stage-15 Stage-1 depends on stages: Stage-6 @@ -596,6 +597,23 @@ Stage: Stage-13 Conditional Operator + Stage: Stage-16 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 + 1 + Stage: Stage-12 Map Reduce Map Operator Tree: @@ -618,13 +636,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-15 Map Reduce Local Work @@ -867,6 +878,7 @@ RUN: Stage-0:MOVE RUN: Stage-13:CONDITIONAL RUN: Stage-3:STATS +RUN: Stage-16:MAPREDLOCAL RUN: Stage-12:MAPRED RUN: Stage-15:MAPREDLOCAL RUN: Stage-6:MAPRED Index: ql/src/test/results/clientpositive/auto_join25.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join25.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join25.q.out (working copy) @@ -19,17 +19,17 @@ PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 PREHOOK: Output: default@dest1 -Execution failed with exit status: 2 +Execution failed with exit status: 3 Obtaining error information Task failed! Task ID: - Stage-4 + Stage-7 Logs: #### A masked pattern was here #### -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask +FAILED: Execution Error, return code 3 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask POSTHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value @@ -45,7 +45,7 @@ POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] RUN: Stage-6:CONDITIONAL -RUN: Stage-4:MAPRED +RUN: Stage-7:MAPREDLOCAL RUN: Stage-1:MAPRED RUN: Stage-0:MOVE RUN: Stage-2:STATS @@ -76,29 +76,29 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@dest_j2 -Execution failed with exit status: 2 +Execution failed with exit status: 3 Obtaining error information Task failed! Task ID: - Stage-10 + Stage-14 Logs: #### A masked pattern was here #### -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask +FAILED: Execution Error, return code 3 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask -Execution failed with exit status: 2 +Execution failed with exit status: 3 Obtaining error information Task failed! Task ID: - Stage-6 + Stage-12 Logs: #### A masked pattern was here #### -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask +FAILED: Execution Error, return code 3 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value @@ -110,10 +110,10 @@ POSTHOOK: Lineage: dest_j2.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j2.value SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ] RUN: Stage-11:CONDITIONAL -RUN: Stage-10:MAPRED +RUN: Stage-14:MAPREDLOCAL RUN: Stage-1:MAPRED RUN: Stage-8:CONDITIONAL -RUN: Stage-6:MAPRED +RUN: Stage-12:MAPREDLOCAL RUN: Stage-2:MAPRED RUN: Stage-0:MOVE RUN: Stage-3:STATS @@ -148,17 +148,17 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@dest_j1 -Execution failed with exit status: 2 +Execution failed with exit status: 3 Obtaining error information Task failed! Task ID: - Stage-4 + Stage-7 Logs: #### A masked pattern was here #### -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask +FAILED: Execution Error, return code 3 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value @@ -172,7 +172,7 @@ POSTHOOK: Lineage: dest_j2.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j2.value SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ] RUN: Stage-6:CONDITIONAL -RUN: Stage-4:MAPRED +RUN: Stage-7:MAPREDLOCAL RUN: Stage-1:MAPRED RUN: Stage-0:MOVE RUN: Stage-2:STATS Index: ql/src/test/results/clientpositive/subq_where_serialization.q.out =================================================================== --- ql/src/test/results/clientpositive/subq_where_serialization.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/subq_where_serialization.q.out (working copy) @@ -4,8 +4,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-2 - Stage-4 has a backup stage: Stage-2 + Stage-5 depends on stages: Stage-1 , consists of Stage-6, Stage-2 + Stage-6 has a backup stage: Stage-2 + Stage-4 depends on stages: Stage-6 Stage-2 Stage-0 is a root stage @@ -55,6 +56,23 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 _col0 (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -83,13 +101,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-2 Map Reduce Index: ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out =================================================================== --- ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out (working copy) @@ -122,10 +122,85 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 500 + partition_columns part + partition_columns.types string + rawDataSize 5312 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {part=1/000000_0=[part=1/000000_0], part=1/000001_0=[part=1/000001_0]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -159,69 +234,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 500 - partition_columns part - partition_columns.types string - rawDataSize 5312 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {part=1/000000_0=[part=1/000000_0], part=1/000001_0=[part=1/000001_0]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketmapjoin8.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin8.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin8.q.out (working copy) @@ -124,10 +124,85 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 3 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -161,69 +236,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 3 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -406,10 +418,85 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name value + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -443,69 +530,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name value - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketmapjoin12.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin12.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin12.q.out (working copy) @@ -158,10 +158,84 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -195,68 +269,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {part=1/srcbucket20.txt=[part=1/srcbucket20.txt], part=1/srcbucket21.txt=[part=1/srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -431,10 +443,77 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + part 1 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_3 + numFiles 2 + numRows 0 + partition_columns part + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_3 + partition_columns part + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_3 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_3 + name: default.srcbucket_mapjoin_part_3 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -467,61 +546,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - part 1 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_3 - numFiles 2 - numRows 0 - partition_columns part - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_3 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_3 - partition_columns part - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_3 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_3 - name: default.srcbucket_mapjoin_part_3 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 687 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/join27.q.out =================================================================== --- ql/src/test/results/clientpositive/join27.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join27.q.out (working copy) @@ -16,7 +16,8 @@ FROM src1 x JOIN src y ON (x.value = y.value) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -26,6 +27,25 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 value (type: string) + 1 value (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -57,15 +77,6 @@ name: default.dest_j1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE Stage: Stage-7 Conditional Operator Index: ql/src/test/results/clientpositive/join36.q.out =================================================================== --- ql/src/test/results/clientpositive/join36.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join36.q.out (working copy) @@ -60,7 +60,8 @@ POSTHOOK: Lineage: tmp2.cnt EXPRESSION [(src)src.null, ] POSTHOOK: Lineage: tmp2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -70,6 +71,25 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {cnt} + 1 {cnt} + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -101,15 +121,6 @@ name: default.dest_j1 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE Stage: Stage-7 Conditional Operator Index: ql/src/test/results/clientpositive/bucketmapjoin3.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin3.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin3.q.out (working copy) @@ -183,7 +183,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6 Stage-5 Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 @@ -193,6 +194,80 @@ Stage-7 depends on stages: Stage-6 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part + partition_columns ds + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part + name: default.srcbucket_mapjoin_part + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -245,69 +320,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part - partition_columns ds - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part - name: default.srcbucket_mapjoin_part - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -776,7 +788,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -786,6 +799,80 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 3062 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns ds + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt], ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket23.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -843,69 +930,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 3062 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns ds - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt], ds=2008-04-08/srcbucket22.txt=[ds=2008-04-08/srcbucket22.txt], ds=2008-04-08/srcbucket23.txt=[ds=2008-04-08/srcbucket23.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/join31.q.out =================================================================== --- ql/src/test/results/clientpositive/join31.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join31.q.out (working copy) @@ -25,12 +25,14 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-8 depends on stages: Stage-1, Stage-5 , consists of Stage-6, Stage-7, Stage-2 - Stage-6 has a backup stage: Stage-2 + Stage-8 depends on stages: Stage-1, Stage-5 , consists of Stage-9, Stage-10, Stage-2 + Stage-9 has a backup stage: Stage-2 + Stage-6 depends on stages: Stage-9 Stage-3 depends on stages: Stage-2, Stage-6, Stage-7 Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 - Stage-7 has a backup stage: Stage-2 + Stage-10 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-10 Stage-2 Stage-5 is a root stage @@ -78,6 +80,23 @@ Stage: Stage-8 Conditional Operator + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -108,13 +127,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-3 Map Reduce @@ -159,6 +171,23 @@ Stage: Stage-4 Stats-Aggr Operator + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -189,13 +218,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME1 - TableScan Stage: Stage-2 Map Reduce Index: ql/src/test/results/clientpositive/join32_lessSize.q.out =================================================================== --- ql/src/test/results/clientpositive/join32_lessSize.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/join32_lessSize.q.out (working copy) @@ -102,12 +102,35 @@ STAGE DEPENDENCIES: - Stage-6 is a root stage - Stage-5 depends on stages: Stage-6 + Stage-8 is a root stage + Stage-6 depends on stages: Stage-8 + Stage-7 depends on stages: Stage-6 + Stage-5 depends on stages: Stage-7 Stage-0 depends on stages: Stage-5 Stage-2 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -146,16 +169,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -250,6 +263,73 @@ Truncated Path -> Alias: /src [y] + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + z + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Alias -> Map Local Operator Tree: + z + TableScan + alias: z + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col5} {_col0} + 1 {value} + keys: + 0 _col1 (type: string) + 1 value (type: string) + Position of Big Table: 0 + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -299,62 +379,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - z - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 11 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 0 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 0 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart - Alias -> Map Local Operator Tree: - z - TableScan - alias: z - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -663,16 +687,41 @@ STAGE DEPENDENCIES: - Stage-11 is a root stage - Stage-10 depends on stages: Stage-11 , consists of Stage-7, Stage-8, Stage-9, Stage-2 - Stage-7 has a backup stage: Stage-2 + Stage-15 is a root stage + Stage-11 depends on stages: Stage-15 + Stage-10 depends on stages: Stage-11 , consists of Stage-12, Stage-13, Stage-14, Stage-2 + Stage-12 has a backup stage: Stage-2 + Stage-7 depends on stages: Stage-12 Stage-0 depends on stages: Stage-2, Stage-7, Stage-8, Stage-9 Stage-3 depends on stages: Stage-0 - Stage-8 has a backup stage: Stage-2 - Stage-9 has a backup stage: Stage-2 + Stage-13 has a backup stage: Stage-2 + Stage-8 depends on stages: Stage-13 + Stage-14 has a backup stage: Stage-2 + Stage-9 depends on stages: Stage-14 Stage-2 STAGE PLANS: + Stage: Stage-15 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 {key} + keys: + 0 value (type: string) + 1 value (type: string) + Position of Big Table: 0 + Stage: Stage-11 Map Reduce Map Operator Tree: @@ -711,16 +760,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -818,6 +857,45 @@ Stage: Stage-10 Conditional Operator + Stage: Stage-12 + Map Reduce Local Work + Alias -> Map Local Tables: + y + Fetch Operator + limit: -1 + z + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + y + TableScan + alias: y + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col4} + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 0 + z + TableScan + alias: z + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col4} + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 0 + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -872,22 +950,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - y - Fetch Operator - limit: -1 - z - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - y - TableScan - alias: y - GatherStats: false - z - TableScan - alias: z - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1033,6 +1095,44 @@ Stats-Aggr Operator #### A masked pattern was here #### + Stage: Stage-13 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + z + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col4} + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 1 + z + TableScan + alias: z + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col4} + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 1 + Stage: Stage-8 Map Reduce Map Operator Tree: @@ -1088,21 +1188,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - z - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan - GatherStats: false - z - TableScan - alias: z - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1217,6 +1302,44 @@ Truncated Path -> Alias: /src [y] + Stage: Stage-14 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col4} + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 + y + TableScan + alias: y + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col4} + 1 {value} + 2 {value} + keys: + 0 _col4 (type: string) + 1 key (type: string) + 2 key (type: string) + Position of Big Table: 2 + Stage: Stage-9 Map Reduce Map Operator Tree: @@ -1272,21 +1395,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan - GatherStats: false - y - TableScan - alias: y - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1832,12 +1940,35 @@ STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-6 depends on stages: Stage-7 + Stage-9 is a root stage + Stage-7 depends on stages: Stage-9 + Stage-8 depends on stages: Stage-7 + Stage-6 depends on stages: Stage-8 Stage-0 depends on stages: Stage-6 Stage-3 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + res:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + res:x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -1880,16 +2011,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - res:x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - res:x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -1984,6 +2105,73 @@ Truncated Path -> Alias: /src [res:y] + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + z + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Alias -> Map Local Operator Tree: + z + TableScan + alias: z + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col1 (type: string) + 1 value (type: string) + Position of Big Table: 0 + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -2033,62 +2221,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - z - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 11 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 0 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 0 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart - Alias -> Map Local Operator Tree: - z - TableScan - alias: z - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2432,12 +2564,35 @@ STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-6 depends on stages: Stage-7 + Stage-9 is a root stage + Stage-7 depends on stages: Stage-9 + Stage-8 depends on stages: Stage-7 + Stage-6 depends on stages: Stage-8 Stage-0 depends on stages: Stage-6 Stage-3 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + res:y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + res:y + TableScan + alias: y + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -2480,16 +2635,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - res:y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - res:y - TableScan - alias: y - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2584,6 +2729,73 @@ Truncated Path -> Alias: /src1 [res:x] + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + z + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Alias -> Map Local Operator Tree: + z + TableScan + alias: z + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col1 (type: string) + 1 value (type: string) + Position of Big Table: 0 + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -2638,62 +2850,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - z - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: hr=11 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - hr 11 - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - numFiles 1 - numRows 0 - partition_columns ds/hr - partition_columns.types string:string - rawDataSize 0 - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - columns key,value - columns.comments defaultdefault - columns.types string:string -#### A masked pattern was here #### - name default.srcpart - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct srcpart { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcpart - name: default.srcpart - Alias -> Map Local Operator Tree: - z - TableScan - alias: z - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2971,12 +3127,33 @@ POSTHOOK: Lineage: dest_j2.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_j2.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-6 depends on stages: Stage-7 + Stage-9 is a root stage + Stage-7 depends on stages: Stage-9 + Stage-8 depends on stages: Stage-7 + Stage-6 depends on stages: Stage-8 Stage-0 depends on stages: Stage-6 Stage-3 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + res:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + res:x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -3006,16 +3183,26 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - res:x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - res:x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + x + TableScan + alias: x + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col1 (type: string) + 1 value (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -3045,15 +3232,6 @@ name: default.dest_j2 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - x - TableScan - alias: x - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Move Operator @@ -3239,12 +3417,33 @@ POSTHOOK: Lineage: dest_j2.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_j2.value SIMPLE [(srcpart)x.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-6 depends on stages: Stage-7 + Stage-9 is a root stage + Stage-7 depends on stages: Stage-9 + Stage-8 depends on stages: Stage-7 + Stage-6 depends on stages: Stage-8 Stage-0 depends on stages: Stage-6 Stage-3 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + res:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + res:x + TableScan + alias: x + Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-7 Map Reduce Map Operator Tree: @@ -3274,16 +3473,26 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - res:x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - res:x - TableScan - alias: x - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + y + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + y + TableScan + alias: y + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} + 1 {value} + keys: + 0 _col1 (type: string) + 1 value (type: string) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -3313,15 +3522,6 @@ name: default.dest_j2 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - y - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - y - TableScan - alias: y - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Move Operator Index: ql/src/test/results/clientpositive/multiMapJoin1.q.out =================================================================== --- ql/src/test/results/clientpositive/multiMapJoin1.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/multiMapJoin1.q.out (working copy) @@ -226,10 +226,44 @@ POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-7 is a root stage + Stage-2 depends on stages: Stage-7 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + firstjoin:smalltbl1 + Fetch Operator + limit: -1 + smalltbl2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + firstjoin:smalltbl1 + TableScan + alias: smalltbl1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + smalltbl2 + TableScan + alias: smalltbl2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -274,22 +308,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - firstjoin:smalltbl1 - Fetch Operator - limit: -1 - smalltbl2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - firstjoin:smalltbl1 - TableScan - alias: smalltbl1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - smalltbl2 - TableScan - alias: smalltbl2 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -346,6 +364,7 @@ POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-7:MAPREDLOCAL RUN: Stage-2:MAPRED 580 PREHOOK: query: -- Now run a query with two-way join, which should be converted into a @@ -381,10 +400,44 @@ POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-7 is a root stage + Stage-2 depends on stages: Stage-7 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + firstjoin:smalltbl1 + Fetch Operator + limit: -1 + smalltbl2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + firstjoin:smalltbl1 + TableScan + alias: smalltbl1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + smalltbl2 + TableScan + alias: smalltbl2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -429,22 +482,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - firstjoin:smalltbl1 - Fetch Operator - limit: -1 - smalltbl2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - firstjoin:smalltbl1 - TableScan - alias: smalltbl1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - smalltbl2 - TableScan - alias: smalltbl2 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -501,6 +538,7 @@ POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-7:MAPREDLOCAL RUN: Stage-2:MAPRED 580 PREHOOK: query: -- Now run a query with two-way join, which should first be converted into a @@ -540,10 +578,44 @@ POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-7 is a root stage + Stage-2 depends on stages: Stage-7 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + firstjoin:smalltbl1 + Fetch Operator + limit: -1 + smalltbl2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + firstjoin:smalltbl1 + TableScan + alias: smalltbl1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + smalltbl2 + TableScan + alias: smalltbl2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {key} + keys: + 0 _col1 (type: string) + 1 value (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -594,22 +666,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - firstjoin:smalltbl1 - Fetch Operator - limit: -1 - smalltbl2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - firstjoin:smalltbl1 - TableScan - alias: smalltbl1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - smalltbl2 - TableScan - alias: smalltbl2 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -669,6 +725,7 @@ POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-7:MAPREDLOCAL RUN: Stage-2:MAPRED 270 10 @@ -865,22 +922,30 @@ POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-20 is a root stage , consists of Stage-18, Stage-19, Stage-5 - Stage-18 has a backup stage: Stage-5 - Stage-17 depends on stages: Stage-5, Stage-18, Stage-19 , consists of Stage-15, Stage-16, Stage-1 - Stage-15 has a backup stage: Stage-1 - Stage-14 depends on stages: Stage-1, Stage-15, Stage-16 , consists of Stage-12, Stage-13, Stage-2 - Stage-12 has a backup stage: Stage-2 - Stage-11 depends on stages: Stage-2, Stage-12, Stage-13 , consists of Stage-9, Stage-10, Stage-3 - Stage-9 has a backup stage: Stage-3 + Stage-20 is a root stage , consists of Stage-27, Stage-28, Stage-5 + Stage-27 has a backup stage: Stage-5 + Stage-18 depends on stages: Stage-27 + Stage-17 depends on stages: Stage-5, Stage-18, Stage-19 , consists of Stage-25, Stage-26, Stage-1 + Stage-25 has a backup stage: Stage-1 + Stage-15 depends on stages: Stage-25 + Stage-14 depends on stages: Stage-1, Stage-15, Stage-16 , consists of Stage-23, Stage-24, Stage-2 + Stage-23 has a backup stage: Stage-2 + Stage-12 depends on stages: Stage-23 + Stage-11 depends on stages: Stage-2, Stage-12, Stage-13 , consists of Stage-21, Stage-22, Stage-3 + Stage-21 has a backup stage: Stage-3 + Stage-9 depends on stages: Stage-21 Stage-4 depends on stages: Stage-3, Stage-9, Stage-10 - Stage-10 has a backup stage: Stage-3 + Stage-22 has a backup stage: Stage-3 + Stage-10 depends on stages: Stage-22 Stage-3 - Stage-13 has a backup stage: Stage-2 + Stage-24 has a backup stage: Stage-2 + Stage-13 depends on stages: Stage-24 Stage-2 - Stage-16 has a backup stage: Stage-1 + Stage-26 has a backup stage: Stage-1 + Stage-16 depends on stages: Stage-26 Stage-1 - Stage-19 has a backup stage: Stage-5 + Stage-28 has a backup stage: Stage-5 + Stage-19 depends on stages: Stage-28 Stage-5 Stage-0 is a root stage @@ -888,6 +953,24 @@ Stage: Stage-20 Conditional Operator + Stage: Stage-27 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:join2:join1:smalltbl1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:join2:join1:smalltbl1 + TableScan + alias: smalltbl1 + HashTable Sink Operator + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + Stage: Stage-18 Map Reduce Map Operator Tree: @@ -914,18 +997,28 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:join2:join1:smalltbl1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:join2:join1:smalltbl1 - TableScan - alias: smalltbl1 Stage: Stage-17 Conditional Operator + Stage: Stage-25 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:join2:smalltbl2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:join2:smalltbl2 + TableScan + alias: smalltbl2 + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) + Stage: Stage-15 Map Reduce Map Operator Tree: @@ -951,18 +1044,28 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:join2:smalltbl2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:join2:smalltbl2 - TableScan - alias: smalltbl2 Stage: Stage-14 Conditional Operator + Stage: Stage-23 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:smalltbl3 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:smalltbl3 + TableScan + alias: smalltbl3 + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 {key} + keys: + 0 _col1 (type: string) + 1 key (type: string) + Stage: Stage-12 Map Reduce Map Operator Tree: @@ -988,18 +1091,28 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:smalltbl3 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:smalltbl3 - TableScan - alias: smalltbl3 Stage: Stage-11 Conditional Operator + Stage: Stage-21 + Map Reduce Local Work + Alias -> Map Local Tables: + smalltbl4 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + smalltbl4 + TableScan + alias: smalltbl4 + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 {key} + keys: + 0 _col2 (type: string) + 1 key (type: string) + Stage: Stage-9 Map Reduce Map Operator Tree: @@ -1029,14 +1142,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - smalltbl4 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - smalltbl4 - TableScan - alias: smalltbl4 Stage: Stage-4 Map Reduce @@ -1064,6 +1169,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-22 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 {key} + keys: + 0 _col2 (type: string) + 1 key (type: string) + Stage: Stage-10 Map Reduce Map Operator Tree: @@ -1094,13 +1216,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-3 Map Reduce @@ -1146,6 +1261,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-24 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:$INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:$INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 {key} + keys: + 0 _col1 (type: string) + 1 key (type: string) + Stage: Stage-13 Map Reduce Map Operator Tree: @@ -1172,13 +1304,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:$INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:$INTNAME - TableScan Stage: Stage-2 Map Reduce @@ -1219,6 +1344,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-26 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:join2:$INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:join2:$INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) + Stage: Stage-16 Map Reduce Map Operator Tree: @@ -1245,13 +1387,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:join2:$INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:join2:$INTNAME - TableScan Stage: Stage-1 Map Reduce @@ -1292,6 +1427,24 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-28 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:join2:join1:bigtbl + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:join2:join1:bigtbl + TableScan + alias: bigtbl + HashTable Sink Operator + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + Stage: Stage-19 Map Reduce Map Operator Tree: @@ -1318,14 +1471,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:join2:join1:bigtbl - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:join2:join1:bigtbl - TableScan - alias: bigtbl Stage: Stage-5 Map Reduce @@ -1462,12 +1607,16 @@ POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] RUN: Stage-20:CONDITIONAL +RUN: Stage-27:MAPREDLOCAL RUN: Stage-18:MAPRED RUN: Stage-17:CONDITIONAL +RUN: Stage-25:MAPREDLOCAL RUN: Stage-15:MAPRED RUN: Stage-14:CONDITIONAL +RUN: Stage-23:MAPREDLOCAL RUN: Stage-12:MAPRED RUN: Stage-11:CONDITIONAL +RUN: Stage-21:MAPREDLOCAL RUN: Stage-9:MAPRED RUN: Stage-4:MAPRED 247580 247580 247580 247580 247580 247580 548662743780 548662743780 @@ -1557,10 +1706,72 @@ POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-4 is a root stage + Stage-13 is a root stage + Stage-4 depends on stages: Stage-13 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-13 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:join2:join1:smalltbl1 + Fetch Operator + limit: -1 + join3:join2:smalltbl2 + Fetch Operator + limit: -1 + join3:smalltbl3 + Fetch Operator + limit: -1 + smalltbl4 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:join2:join1:smalltbl1 + TableScan + alias: smalltbl1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + join3:join2:smalltbl2 + TableScan + alias: smalltbl2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) + join3:smalltbl3 + TableScan + alias: smalltbl3 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 {key} + keys: + 0 _col1 (type: string) + 1 key (type: string) + smalltbl4 + TableScan + alias: smalltbl4 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 {key} + keys: + 0 _col2 (type: string) + 1 key (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -1638,36 +1849,6 @@ value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:join2:join1:smalltbl1 - Fetch Operator - limit: -1 - join3:join2:smalltbl2 - Fetch Operator - limit: -1 - join3:smalltbl3 - Fetch Operator - limit: -1 - smalltbl4 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:join2:join1:smalltbl1 - TableScan - alias: smalltbl1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - join3:join2:smalltbl2 - TableScan - alias: smalltbl2 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - join3:smalltbl3 - TableScan - alias: smalltbl3 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - smalltbl4 - TableScan - alias: smalltbl4 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) @@ -1779,6 +1960,7 @@ POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-13:MAPREDLOCAL RUN: Stage-4:MAPRED 247580 247580 247580 247580 247580 247580 548662743780 548662743780 PREHOOK: query: -- Enable noconditionaltask and set the size of hive.auto.convert.join.noconditionaltask.size @@ -1869,11 +2051,46 @@ POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-11 is a root stage - Stage-4 depends on stages: Stage-11 + Stage-14 is a root stage + Stage-11 depends on stages: Stage-14 + Stage-13 depends on stages: Stage-11 + Stage-4 depends on stages: Stage-13 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-14 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:join2:join1:smalltbl1 + Fetch Operator + limit: -1 + join3:join2:smalltbl2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:join2:join1:smalltbl1 + TableScan + alias: smalltbl1 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + join3:join2:smalltbl2 + TableScan + alias: smalltbl2 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) + Stage: Stage-11 Map Reduce Map Operator Tree: @@ -1918,23 +2135,40 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:join2:join1:smalltbl1 - Fetch Operator - limit: -1 - join3:join2:smalltbl2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:join2:join1:smalltbl1 - TableScan - alias: smalltbl1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - join3:join2:smalltbl2 - TableScan - alias: smalltbl2 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Stage: Stage-13 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:smalltbl3 + Fetch Operator + limit: -1 + smalltbl4 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:smalltbl3 + TableScan + alias: smalltbl3 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 {key} + keys: + 0 _col1 (type: string) + 1 key (type: string) + smalltbl4 + TableScan + alias: smalltbl4 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 {key} + keys: + 0 _col2 (type: string) + 1 key (type: string) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -1980,22 +2214,6 @@ value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:smalltbl3 - Fetch Operator - limit: -1 - smalltbl4 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:smalltbl3 - TableScan - alias: smalltbl3 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - smalltbl4 - TableScan - alias: smalltbl4 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4), sum(VALUE._col5), sum(VALUE._col6), sum(VALUE._col7) @@ -2107,7 +2325,9 @@ POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +RUN: Stage-14:MAPREDLOCAL RUN: Stage-11:MAPRED +RUN: Stage-13:MAPREDLOCAL RUN: Stage-4:MAPRED 247580 247580 247580 247580 247580 247580 548662743780 548662743780 PREHOOK: query: -- Enable noconditionaltask and but set the size of hive.auto.convert.join.noconditionaltask.size @@ -2194,22 +2414,30 @@ POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-20 is a root stage , consists of Stage-18, Stage-19, Stage-5 - Stage-18 has a backup stage: Stage-5 - Stage-17 depends on stages: Stage-5, Stage-18, Stage-19 , consists of Stage-15, Stage-16, Stage-1 - Stage-15 has a backup stage: Stage-1 - Stage-14 depends on stages: Stage-1, Stage-15, Stage-16 , consists of Stage-12, Stage-13, Stage-2 - Stage-12 has a backup stage: Stage-2 - Stage-11 depends on stages: Stage-2, Stage-12, Stage-13 , consists of Stage-9, Stage-10, Stage-3 - Stage-9 has a backup stage: Stage-3 + Stage-20 is a root stage , consists of Stage-27, Stage-28, Stage-5 + Stage-27 has a backup stage: Stage-5 + Stage-18 depends on stages: Stage-27 + Stage-17 depends on stages: Stage-5, Stage-18, Stage-19 , consists of Stage-25, Stage-26, Stage-1 + Stage-25 has a backup stage: Stage-1 + Stage-15 depends on stages: Stage-25 + Stage-14 depends on stages: Stage-1, Stage-15, Stage-16 , consists of Stage-23, Stage-24, Stage-2 + Stage-23 has a backup stage: Stage-2 + Stage-12 depends on stages: Stage-23 + Stage-11 depends on stages: Stage-2, Stage-12, Stage-13 , consists of Stage-21, Stage-22, Stage-3 + Stage-21 has a backup stage: Stage-3 + Stage-9 depends on stages: Stage-21 Stage-4 depends on stages: Stage-3, Stage-9, Stage-10 - Stage-10 has a backup stage: Stage-3 + Stage-22 has a backup stage: Stage-3 + Stage-10 depends on stages: Stage-22 Stage-3 - Stage-13 has a backup stage: Stage-2 + Stage-24 has a backup stage: Stage-2 + Stage-13 depends on stages: Stage-24 Stage-2 - Stage-16 has a backup stage: Stage-1 + Stage-26 has a backup stage: Stage-1 + Stage-16 depends on stages: Stage-26 Stage-1 - Stage-19 has a backup stage: Stage-5 + Stage-28 has a backup stage: Stage-5 + Stage-19 depends on stages: Stage-28 Stage-5 Stage-0 is a root stage @@ -2217,6 +2445,24 @@ Stage: Stage-20 Conditional Operator + Stage: Stage-27 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:join2:join1:smalltbl1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:join2:join1:smalltbl1 + TableScan + alias: smalltbl1 + HashTable Sink Operator + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + Stage: Stage-18 Map Reduce Map Operator Tree: @@ -2243,18 +2489,28 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:join2:join1:smalltbl1 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:join2:join1:smalltbl1 - TableScan - alias: smalltbl1 Stage: Stage-17 Conditional Operator + Stage: Stage-25 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:join2:smalltbl2 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:join2:smalltbl2 + TableScan + alias: smalltbl2 + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) + Stage: Stage-15 Map Reduce Map Operator Tree: @@ -2280,18 +2536,28 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:join2:smalltbl2 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:join2:smalltbl2 - TableScan - alias: smalltbl2 Stage: Stage-14 Conditional Operator + Stage: Stage-23 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:smalltbl3 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:smalltbl3 + TableScan + alias: smalltbl3 + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 {key} + keys: + 0 _col1 (type: string) + 1 key (type: string) + Stage: Stage-12 Map Reduce Map Operator Tree: @@ -2317,18 +2583,28 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:smalltbl3 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:smalltbl3 - TableScan - alias: smalltbl3 Stage: Stage-11 Conditional Operator + Stage: Stage-21 + Map Reduce Local Work + Alias -> Map Local Tables: + smalltbl4 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + smalltbl4 + TableScan + alias: smalltbl4 + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 {key} + keys: + 0 _col2 (type: string) + 1 key (type: string) + Stage: Stage-9 Map Reduce Map Operator Tree: @@ -2358,14 +2634,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - smalltbl4 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - smalltbl4 - TableScan - alias: smalltbl4 Stage: Stage-4 Map Reduce @@ -2393,6 +2661,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-22 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col6} + 1 {key} + keys: + 0 _col2 (type: string) + 1 key (type: string) + Stage: Stage-10 Map Reduce Map Operator Tree: @@ -2423,13 +2708,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan Stage: Stage-3 Map Reduce @@ -2475,6 +2753,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-24 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:$INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:$INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} {_col5} + 1 {key} + keys: + 0 _col1 (type: string) + 1 key (type: string) + Stage: Stage-13 Map Reduce Map Operator Tree: @@ -2501,13 +2796,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:$INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:$INTNAME - TableScan Stage: Stage-2 Map Reduce @@ -2548,6 +2836,23 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-26 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:join2:$INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:join2:$INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {_col0} {_col1} {_col2} {_col3} {_col4} + 1 {key} + keys: + 0 _col3 (type: string) + 1 value (type: string) + Stage: Stage-16 Map Reduce Map Operator Tree: @@ -2574,13 +2879,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:join2:$INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:join2:$INTNAME - TableScan Stage: Stage-1 Map Reduce @@ -2621,6 +2919,24 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-28 + Map Reduce Local Work + Alias -> Map Local Tables: + join3:join2:join1:bigtbl + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + join3:join2:join1:bigtbl + TableScan + alias: bigtbl + HashTable Sink Operator + condition expressions: + 0 {key1} {key2} {value} + 1 {key} + keys: + 0 key1 (type: string) + 1 key (type: string) + Stage: Stage-19 Map Reduce Map Operator Tree: @@ -2647,14 +2963,6 @@ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - join3:join2:join1:bigtbl - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - join3:join2:join1:bigtbl - TableScan - alias: bigtbl Stage: Stage-5 Map Reduce @@ -2791,12 +3099,16 @@ POSTHOOK: Lineage: smalltbl4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] RUN: Stage-20:CONDITIONAL +RUN: Stage-27:MAPREDLOCAL RUN: Stage-18:MAPRED RUN: Stage-17:CONDITIONAL +RUN: Stage-25:MAPREDLOCAL RUN: Stage-15:MAPRED RUN: Stage-14:CONDITIONAL +RUN: Stage-23:MAPREDLOCAL RUN: Stage-12:MAPRED RUN: Stage-11:CONDITIONAL +RUN: Stage-21:MAPREDLOCAL RUN: Stage-9:MAPRED RUN: Stage-4:MAPRED 247580 247580 247580 247580 247580 247580 548662743780 548662743780 Index: ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out (working copy) @@ -597,9 +597,11 @@ STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-4, Stage-1 - Stage-3 has a backup stage: Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-1 Stage-0 is a root stage @@ -607,6 +609,119 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 226 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 226 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -634,108 +749,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 226 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 226 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -962,6 +975,121 @@ GatherStats: false MultiFileSpray: false + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=2008-04-08 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_big + name: default.bucket_big + Partition + base file name: ds=2008-04-09 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_big + name: default.bucket_big + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -989,110 +1117,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: ds=2008-04-08 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_big - name: default.bucket_big - Partition - base file name: ds=2008-04-09 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_big - name: default.bucket_big - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/correlationoptimizer6.q.out =================================================================== --- ql/src/test/results/clientpositive/correlationoptimizer6.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/correlationoptimizer6.q.out (working copy) @@ -503,11 +503,45 @@ ON xx.key=yy.key ORDER BY xx.key, xx.cnt, yy.key, yy.cnt POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-9 is a root stage + Stage-2 depends on stages: Stage-9 Stage-3 depends on stages: Stage-2 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + xx:x + Fetch Operator + limit: -1 + yy:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + xx:x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + yy:x + TableScan + alias: x + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -573,22 +607,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - xx:x - Fetch Operator - limit: -1 - yy:x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - xx:x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE - yy:x - TableScan - alias: x - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator Statistics: Num rows: 65 Data size: 6630 Basic stats: COMPLETE Column stats: NONE @@ -3886,12 +3904,33 @@ ON xx.key=yy.key ORDER BY xx.key, xx.cnt, yy.key, yy.value, yy.cnt POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-3 depends on stages: Stage-2 + Stage-11 is a root stage + Stage-2 depends on stages: Stage-11 + Stage-10 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-10 Stage-4 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-11 + Map Reduce Local Work + Alias -> Map Local Tables: + yy:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + yy:x + TableScan + alias: x + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -3927,15 +3966,6 @@ value expressions: _col2 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - yy:x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - yy:x - TableScan - alias: x - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -3954,6 +3984,25 @@ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-10 + Map Reduce Local Work + Alias -> Map Local Tables: + xx:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + xx:x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -3996,15 +4045,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - xx:x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - xx:x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator Statistics: Num rows: 33 Data size: 3382 Basic stats: COMPLETE Column stats: NONE Index: ql/src/test/results/clientpositive/select_transform_hint.q.out =================================================================== --- ql/src/test/results/clientpositive/select_transform_hint.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/select_transform_hint.q.out (working copy) @@ -11,10 +11,30 @@ on a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -52,15 +72,6 @@ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator Index: ql/src/test/results/clientpositive/auto_join2.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_join2.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_join2.q.out (working copy) @@ -14,11 +14,45 @@ INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-6 is a root stage + Stage-8 is a root stage + Stage-6 depends on stages: Stage-8 Stage-0 depends on stages: Stage-6 Stage-3 depends on stages: Stage-0 STAGE PLANS: + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + src1 + Fetch Operator + limit: -1 + src3 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + src1 + TableScan + alias: src1 + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + src3 + TableScan + alias: src3 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {_col0} + 1 {value} + keys: + 0 (_col0 + _col4) (type: double) + 1 UDFToDouble(key) (type: double) + Stage: Stage-6 Map Reduce Map Operator Tree: @@ -61,22 +95,6 @@ name: default.dest_j2 Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - src1 - Fetch Operator - limit: -1 - src3 - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - src1 - TableScan - alias: src1 - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - src3 - TableScan - alias: src3 - Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Move Operator Index: ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out (working copy) @@ -278,17 +278,37 @@ POSTHOOK: Lineage: test_table3 PARTITION(ds=1).key SIMPLE [(test_table1)a.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: test_table3 PARTITION(ds=1).value EXPRESSION [(test_table1)a.FieldSchema(name:value, type:string, comment:null), (test_table2)b.FieldSchema(name:value, type:string, comment:null), ] STAGE DEPENDENCIES: - Stage-6 is a root stage , consists of Stage-4, Stage-5, Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-0 depends on stages: Stage-1, Stage-4, Stage-5 Stage-3 depends on stages: Stage-0 - Stage-5 has a backup stage: Stage-1 + Stage-8 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-8 Stage-1 STAGE PLANS: Stage: Stage-6 Conditional Operator + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -314,14 +334,6 @@ value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b Reduce Operator Tree: Extract File Output Operator @@ -347,6 +359,24 @@ Stage: Stage-3 Stats-Aggr Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-5 Map Reduce Map Operator Tree: @@ -372,14 +402,6 @@ value expressions: _col0 (type: int), _col1 (type: string) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a Reduce Operator Tree: Extract File Output Operator Index: ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out (working copy) @@ -342,9 +342,11 @@ STAGE DEPENDENCIES: - Stage-5 is a root stage , consists of Stage-3, Stage-4, Stage-1 - Stage-3 has a backup stage: Stage-1 - Stage-4 has a backup stage: Stage-1 + Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1 + Stage-6 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-6 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 Stage-1 Stage-0 is a root stage @@ -352,6 +354,73 @@ Stage: Stage-5 Conditional Operator + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 226 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_small + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_small { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_small + name: default.bucket_small + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 0 + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -379,62 +448,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 226 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_small - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_small { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_small - name: default.bucket_small - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -614,6 +627,121 @@ GatherStats: false MultiFileSpray: false + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Partition Description: + Partition + base file name: ds=2008-04-08 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_big + name: default.bucket_big + Partition + base file name: ds=2008-04-09 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 2750 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.bucket_big + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket_big { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket_big + name: default.bucket_big + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Stage: Stage-4 Map Reduce Map Operator Tree: @@ -641,110 +769,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Partition Description: - Partition - base file name: ds=2008-04-08 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_big - name: default.bucket_big - Partition - base file name: ds=2008-04-09 - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 2750 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.bucket_big - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket_big { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket_big - name: default.bucket_big - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out (working copy) @@ -132,7 +132,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6 Stage-5 Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 @@ -142,6 +143,125 @@ Stage-7 depends on stages: Stage-6 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 3062 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns ds + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + numFiles 2 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 3062 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part_2 + partition_columns ds + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part_2 + name: default.srcbucket_mapjoin_part_2 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 58 Data size: 6124 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt, ds=2008-04-09/srcbucket22.txt], srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt, ds=2008-04-09/srcbucket23.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -194,114 +314,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 3062 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns ds - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-09 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - numFiles 2 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 3062 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part_2 - partition_columns ds - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part_2 - name: default.srcbucket_mapjoin_part_2 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 58 Data size: 6124 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {srcbucket20.txt=[ds=2008-04-08/srcbucket22.txt, ds=2008-04-09/srcbucket22.txt], srcbucket21.txt=[ds=2008-04-08/srcbucket23.txt, ds=2008-04-09/srcbucket23.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/correlationoptimizer1.q.out =================================================================== --- ql/src/test/results/clientpositive/correlationoptimizer1.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/correlationoptimizer1.q.out (working copy) @@ -298,11 +298,31 @@ GROUP BY x.key) tmp POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 Stage-3 depends on stages: Stage-2 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + tmp:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + tmp:x + TableScan + alias: x + Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Stage: Stage-2 Map Reduce Map Operator Tree: @@ -338,15 +358,6 @@ value expressions: _col1 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - tmp:x - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - tmp:x - TableScan - alias: x - Statistics: Num rows: 2 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) Index: ql/src/test/results/clientpositive/bucket_map_join_2.q.out =================================================================== --- ql/src/test/results/clientpositive/bucket_map_join_2.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucket_map_join_2.q.out (working copy) @@ -102,10 +102,32 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 0 Data size: 21 Basic stats: PARTIAL Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string), value (type: string) + 1 key (type: string), value (type: string) + Position of Big Table: 0 + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -138,16 +160,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 0 Data size: 21 Basic stats: PARTIAL Column stats: NONE - GatherStats: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/auto_sortmerge_join_14.q.out =================================================================== --- ql/src/test/results/clientpositive/auto_sortmerge_join_14.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/auto_sortmerge_join_14.q.out (working copy) @@ -47,8 +47,9 @@ POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-4 is a root stage , consists of Stage-3, Stage-1 - Stage-3 has a backup stage: Stage-1 + Stage-4 is a root stage , consists of Stage-5, Stage-1 + Stage-5 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-5 Stage-1 Stage-0 is a root stage @@ -56,6 +57,24 @@ Stage: Stage-4 Conditional Operator + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -80,14 +99,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - b - TableScan - alias: b Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -189,8 +200,9 @@ POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] STAGE DEPENDENCIES: - Stage-4 is a root stage , consists of Stage-3, Stage-1 - Stage-3 has a backup stage: Stage-1 + Stage-4 is a root stage , consists of Stage-5, Stage-1 + Stage-5 has a backup stage: Stage-1 + Stage-3 depends on stages: Stage-5 Stage-1 Stage-0 is a root stage @@ -198,6 +210,24 @@ Stage: Stage-4 Conditional Operator + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: int) + 1 key (type: int) + Stage: Stage-3 Map Reduce Map Operator Tree: @@ -222,14 +252,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) Index: ql/src/test/results/clientpositive/bucketcontext_5.q.out =================================================================== --- ql/src/test/results/clientpositive/bucketcontext_5.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/bucketcontext_5.q.out (working copy) @@ -104,10 +104,39 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 Stage-0 is a root stage STAGE PLANS: + Stage: Stage-3 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 + 1 + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {srcsortbucket1outof4.txt=[srcsortbucket1outof4.txt, srcsortbucket3outof4.txt], srcsortbucket2outof4.txt=[srcsortbucket2outof4.txt, srcsortbucket4outof4.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -141,23 +170,6 @@ value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {srcsortbucket1outof4.txt=[srcsortbucket1outof4.txt, srcsortbucket3outof4.txt], srcsortbucket2outof4.txt=[srcsortbucket2outof4.txt, srcsortbucket4outof4.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: Index: ql/src/test/results/clientpositive/stats11.q.out =================================================================== --- ql/src/test/results/clientpositive/stats11.q.out (revision 1584069) +++ ql/src/test/results/clientpositive/stats11.q.out (working copy) @@ -358,7 +358,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-8 depends on stages: Stage-1 , consists of Stage-5, Stage-4, Stage-6 Stage-5 Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 @@ -368,6 +369,80 @@ Stage-7 depends on stages: Stage-6 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Partition Description: + Partition + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + properties: + COLUMN_STATS_ACCURATE true + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part + numFiles 4 + numRows 0 + partition_columns ds + partition_columns.types string + rawDataSize 0 + serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 4 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.srcbucket_mapjoin_part + partition_columns ds + partition_columns.types string + serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_part + name: default.srcbucket_mapjoin_part + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 0 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + b {srcbucket20.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], srcbucket21.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -420,69 +495,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - b - Fetch Operator - limit: -1 - Partition Description: - Partition - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - partition values: - ds 2008-04-08 - properties: - COLUMN_STATS_ACCURATE true - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part - numFiles 4 - numRows 0 - partition_columns ds - partition_columns.types string - rawDataSize 0 - serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 4 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string -#### A masked pattern was here #### - name default.srcbucket_mapjoin_part - partition_columns ds - partition_columns.types string - serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_part - name: default.srcbucket_mapjoin_part - Alias -> Map Local Operator Tree: - b - TableScan - alias: b - Statistics: Num rows: 55 Data size: 5812 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - b {srcbucket20.txt=[ds=2008-04-08/srcbucket20.txt, ds=2008-04-08/srcbucket22.txt], srcbucket21.txt=[ds=2008-04-08/srcbucket21.txt, ds=2008-04-08/srcbucket23.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -939,7 +951,8 @@ STAGE DEPENDENCIES: - Stage-1 is a root stage + Stage-9 is a root stage + Stage-1 depends on stages: Stage-9 Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 @@ -949,6 +962,34 @@ Stage-6 depends on stages: Stage-5 STAGE PLANS: + Stage: Stage-9 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: int) + 1 key (type: int) + Position of Big Table: 1 + Bucket Mapjoin Context: + Alias Bucket Base File Name Mapping: + a {ds=2008-04-08/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt]} + Alias Bucket File Name Mapping: +#### A masked pattern was here #### + Alias Bucket Output File Name Mapping: +#### A masked pattern was here #### + Stage: Stage-1 Map Reduce Map Operator Tree: @@ -1006,23 +1047,6 @@ MultiFileSpray: false Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - a - TableScan - alias: a - Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Bucket Mapjoin Context: - Alias Bucket Base File Name Mapping: - a {ds=2008-04-08/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-08/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-08/srcbucket23.txt=[srcbucket21.txt]} - Alias Bucket File Name Mapping: -#### A masked pattern was here #### - Alias Bucket Output File Name Mapping: -#### A masked pattern was here #### Path -> Alias: #### A masked pattern was here #### Path -> Partition: