diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java index 95f43d4..b6995c9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java @@ -31,6 +31,7 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.core.Sort; @@ -285,9 +286,24 @@ private QueryBlockInfo convertSource(RelNode r) throws CalciteSemanticException s = new Schema(left.schema, right.schema); ASTNode cond = join.getCondition().accept(new RexVisitor(s)); boolean semiJoin = join instanceof SemiJoin; - ast = ASTBuilder.join(left.ast, right.ast, join.getJoinType(), cond, semiJoin); - if (semiJoin) + if (join.getRight() instanceof Join) { + // Invert join inputs; this is done because otherwise the SemanticAnalyzer + // methods to merge joins will not kick in + JoinRelType type; + if (join.getJoinType() == JoinRelType.LEFT) { + type = JoinRelType.RIGHT; + } else if (join.getJoinType() == JoinRelType.RIGHT) { + type = JoinRelType.LEFT; + } else { + type = join.getJoinType(); + } + ast = ASTBuilder.join(right.ast, left.ast, type, cond, semiJoin); + } else { + ast = ASTBuilder.join(left.ast, right.ast, join.getJoinType(), cond, semiJoin); + } + if (semiJoin) { s = left.schema; + } } else if (r instanceof Union) { RelNode leftInput = ((Union) r).getInput(0); RelNode rightInput = ((Union) r).getInput(1); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java index d8be7ff..5f6be9e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java @@ -242,7 +242,15 @@ private static boolean validJoinParent(RelNode joinNode, RelNode parent) { boolean validParent = true; if (parent instanceof Join) { - if (((Join) parent).getRight() == joinNode) { + // In Hive AST, right child of join cannot be another join, + // thus we need to introduce a project on top of it. + // But we only need the additional project if the left child + // is another join too; if it is not, ASTConverter will swap + // the join inputs, leaving the join operator on the left. + // This will help triggering multijoin recognition methods that + // are embedded in SemanticAnalyzer. + if (((Join) parent).getRight() == joinNode && + (((Join) parent).getLeft() instanceof Join) ) { validParent = false; } } else if (parent instanceof SetOp) { @@ -255,7 +263,7 @@ private static boolean validJoinParent(RelNode joinNode, RelNode parent) { private static boolean validFilterParent(RelNode filterNode, RelNode parent) { boolean validParent = true; - // TOODO: Verify GB having is not a seperate filter (if so we shouldn't + // TODO: Verify GB having is not a separate filter (if so we shouldn't // introduce derived table) if (parent instanceof Filter || parent instanceof Join || parent instanceof SetOp) { diff --git ql/src/test/results/clientpositive/auto_join13.q.out ql/src/test/results/clientpositive/auto_join13.q.out index c5d6b44..952dbf8 100644 --- ql/src/test/results/clientpositive/auto_join13.q.out +++ ql/src/test/results/clientpositive/auto_join13.q.out @@ -21,18 +21,18 @@ JOIN ON src1.c1 + src2.c3 = src3.c5 AND src3.c5 < 200 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-2 depends on stages: Stage-7 - Stage-0 depends on stages: Stage-2 + Stage-8 is a root stage + Stage-3 depends on stages: Stage-8 + Stage-0 depends on stages: Stage-3 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-8 Map Reduce Local Work Alias -> Map Local Tables: $hdt$_0:$hdt$_0:src Fetch Operator limit: -1 - $hdt$_0:$hdt$_1:$hdt$_1:src + $hdt$_0:$hdt$_1:src Fetch Operator limit: -1 Alias -> Map Local Operator Tree: @@ -49,9 +49,9 @@ STAGE PLANS: Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 UDFToDouble(_col0) (type: double) - 1 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) - $hdt$_0:$hdt$_1:$hdt$_1:src + 0 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) + 1 UDFToDouble(_col0) (type: double) + $hdt$_0:$hdt$_1:src TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -67,7 +67,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) - Stage: Stage-2 + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -95,12 +95,12 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 UDFToDouble(_col0) (type: double) - 1 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) - outputColumnNames: _col2, _col3 + 0 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) + 1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col1, _col2 Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: hash(_col3,_col2) (type: int) + expressions: hash(_col2,_col1) (type: int) outputColumnNames: _col0 Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE Group By Operator diff --git ql/src/test/results/clientpositive/auto_join_without_localtask.q.out ql/src/test/results/clientpositive/auto_join_without_localtask.q.out index ab77bfe..3d0067b 100644 --- ql/src/test/results/clientpositive/auto_join_without_localtask.q.out +++ ql/src/test/results/clientpositive/auto_join_without_localtask.q.out @@ -654,32 +654,32 @@ POSTHOOK: query: explain select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 order by a.key, a.value limit 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-10 is a root stage , consists of Stage-13, Stage-14, Stage-3 - Stage-13 has a backup stage: Stage-3 - Stage-8 depends on stages: Stage-13 - Stage-7 depends on stages: Stage-3, Stage-8, Stage-9 , consists of Stage-5, Stage-12, Stage-1 - Stage-5 has a backup stage: Stage-1 - Stage-2 depends on stages: Stage-1, Stage-5, Stage-6 - Stage-12 has a backup stage: Stage-1 + Stage-11 is a root stage , consists of Stage-14, Stage-15, Stage-1 + Stage-14 has a backup stage: Stage-1 + Stage-9 depends on stages: Stage-14 + Stage-8 depends on stages: Stage-1, Stage-9, Stage-10 , consists of Stage-12, Stage-7, Stage-2 + Stage-12 has a backup stage: Stage-2 Stage-6 depends on stages: Stage-12 + Stage-3 depends on stages: Stage-2, Stage-6, Stage-7 + Stage-7 has a backup stage: Stage-2 + Stage-2 + Stage-15 has a backup stage: Stage-1 + Stage-10 depends on stages: Stage-15 Stage-1 - Stage-14 has a backup stage: Stage-3 - Stage-9 depends on stages: Stage-14 - Stage-3 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 STAGE PLANS: - Stage: Stage-10 + Stage: Stage-11 Conditional Operator - Stage: Stage-13 + Stage: Stage-14 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_1:$hdt$_2:a + $hdt$_2:a Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_1:$hdt$_2:a + $hdt$_2:a TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -695,7 +695,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) - Stage: Stage-8 + Stage: Stage-9 Map Reduce Map Operator Tree: TableScan @@ -725,12 +725,17 @@ STAGE PLANS: Local Work: Map Reduce Local Work - Stage: Stage-7 + Stage: Stage-8 Conditional Operator - Stage: Stage-5 - Map Reduce - Map Operator Tree: + Stage: Stage-12 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:a TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -741,35 +746,33 @@ STAGE PLANS: expressions: value (type: string) outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 + HashTable Sink Operator keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + 0 _col1 (type: string) + 1 _col0 (type: string) + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Alias -> Map Local Tables: - $INTNAME - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $INTNAME - TableScan - Stage: Stage-2 + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -793,14 +796,9 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-12 - Map Reduce Local Work - Alias -> Map Local Tables: - $hdt$_0:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - $hdt$_0:a + Stage: Stage-7 + Map Reduce + Map Operator Tree: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -811,40 +809,41 @@ STAGE PLANS: expressions: value (type: string) outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator + Map Join Operator + condition map: + Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan - Stage: Stage-1 + Stage: Stage-2 Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -859,41 +858,30 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col2 + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-14 + Stage: Stage-15 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_1:$hdt$_1:a + $hdt$_1:a Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_1:$hdt$_1:a + $hdt$_1:a TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -909,7 +897,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) - Stage: Stage-9 + Stage: Stage-10 Map Reduce Map Operator Tree: TableScan @@ -939,7 +927,7 @@ STAGE PLANS: Local Work: Map Reduce Local Work - Stage: Stage-3 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan @@ -1003,13 +991,13 @@ POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -RUN: Stage-10:CONDITIONAL -RUN: Stage-13:MAPREDLOCAL -RUN: Stage-8:MAPRED -RUN: Stage-7:CONDITIONAL +RUN: Stage-11:CONDITIONAL +RUN: Stage-14:MAPREDLOCAL +RUN: Stage-9:MAPRED +RUN: Stage-8:CONDITIONAL RUN: Stage-12:MAPREDLOCAL RUN: Stage-6:MAPRED -RUN: Stage-2:MAPRED +RUN: Stage-3:MAPRED 103 val_103 103 val_103 103 val_103 @@ -1064,13 +1052,13 @@ select a.* from src a join src b on a.key=b.key join src c on a.value=c.value wh POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -RUN: Stage-10:CONDITIONAL -RUN: Stage-13:MAPREDLOCAL -RUN: Stage-3:MAPRED -RUN: Stage-7:CONDITIONAL -RUN: Stage-12:MAPREDLOCAL +RUN: Stage-11:CONDITIONAL +RUN: Stage-14:MAPREDLOCAL RUN: Stage-1:MAPRED +RUN: Stage-8:CONDITIONAL +RUN: Stage-12:MAPREDLOCAL RUN: Stage-2:MAPRED +RUN: Stage-3:MAPRED 103 val_103 103 val_103 103 val_103 diff --git ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out index f4b25ae..79b4650 100644 --- ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out +++ ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out @@ -1127,8 +1127,7 @@ select count(*) from POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 @@ -1144,57 +1143,21 @@ STAGE PLANS: expressions: key (type: int) outputColumnNames: _col0 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < 6) (type: boolean) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 + Inner Join 0 to 2 keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0 - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - sort order: - value expressions: _col0 (type: bigint) + 2 _col0 (type: int) + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git ql/src/test/results/clientpositive/correlationoptimizer6.q.out ql/src/test/results/clientpositive/correlationoptimizer6.q.out index 38e51fa..85e447c 100644 --- ql/src/test/results/clientpositive/correlationoptimizer6.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer6.q.out @@ -1787,14 +1787,13 @@ JOIN ON zz.key=yy.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2, Stage-5 - Stage-5 is a root stage - Stage-0 depends on stages: Stage-3 + Stage-3 is a root stage + Stage-4 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -1849,7 +1848,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-2 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan @@ -1873,45 +1872,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - TableScan - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-5 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan @@ -1944,21 +1905,35 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 + Inner Join 0 to 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + 2 _col0 (type: string) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2069,20 +2044,20 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan - alias: zz - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + alias: xx + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE TableScan alias: zz Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -2099,23 +2074,46 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan - alias: xx - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + alias: zz + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator Statistics: Num rows: 763 Data size: 8067 Basic stats: COMPLETE Column stats: NONE + Mux Operator + Statistics: Num rows: 1527 Data size: 16134 Basic stats: COMPLETE Column stats: NONE + Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Join Operator condition map: Inner Join 0 to 1 @@ -2133,17 +2131,19 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Mux Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1527 Data size: 16134 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 + Inner Join 0 to 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 + 2 _col0 (type: string) + outputColumnNames: _col1, _col2, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col1 (type: bigint) + expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator @@ -2153,35 +2153,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Mux Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/join13.q.out ql/src/test/results/clientpositive/join13.q.out index 3f3f7e5..3b921b9 100644 --- ql/src/test/results/clientpositive/join13.q.out +++ ql/src/test/results/clientpositive/join13.q.out @@ -25,12 +25,12 @@ JOIN ON src1.c1 + src2.c3 = src3.c5 AND src3.c5 < 200 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-1 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-1 + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: - Stage: Stage-2 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan @@ -83,10 +83,17 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-1 + Stage: Stage-2 Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + key expressions: (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) + sort order: + + Map-reduce partition columns: (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) + Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string) + TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -101,24 +108,17 @@ STAGE PLANS: sort order: + Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - TableScan - Reduce Output Operator - key expressions: (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) - sort order: + - Map-reduce partition columns: (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 UDFToDouble(_col0) (type: double) - 1 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) - outputColumnNames: _col2, _col3 + 0 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) + 1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col1, _col2 Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col2 (type: string) + expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/join32.q.out ql/src/test/results/clientpositive/join32.q.out index 49e302a..5795669 100644 --- ql/src/test/results/clientpositive/join32.q.out +++ ql/src/test/results/clientpositive/join32.q.out @@ -100,19 +100,19 @@ TOK_QUERY STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-5 depends on stages: Stage-7 - Stage-0 depends on stages: Stage-5 - Stage-2 depends on stages: Stage-0 + Stage-8 is a root stage + Stage-6 depends on stages: Stage-8 + Stage-0 depends on stages: Stage-6 + Stage-3 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-8 Map Reduce Local Work Alias -> Map Local Tables: $hdt$_0:y Fetch Operator limit: -1 - $hdt$_1:$hdt$_2:x + $hdt$_2:x Fetch Operator limit: -1 Alias -> Map Local Operator Tree: @@ -131,10 +131,10 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - Position of Big Table: 1 - $hdt$_1:$hdt$_2:x + 0 _col3 (type: string) + 1 _col0 (type: string) + Position of Big Table: 0 + $hdt$_2:x TableScan alias: x Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE @@ -153,7 +153,7 @@ STAGE PLANS: 1 _col1 (type: string) Position of Big Table: 0 - Stage: Stage-5 + Stage: Stage-6 Map Reduce Map Operator Tree: TableScan @@ -181,13 +181,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - outputColumnNames: _col1, _col2, _col5 - Position of Big Table: 1 + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 + Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col5 (type: string), _col2 (type: string), _col1 (type: string) + expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -356,7 +356,7 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [$hdt$_1:$hdt$_1:z] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_1:z] Stage: Stage-0 Move Operator @@ -380,7 +380,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 - Stage: Stage-2 + Stage: Stage-3 Stats-Aggr Operator #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/join32_lessSize.q.out ql/src/test/results/clientpositive/join32_lessSize.q.out index 736a912..c027dba 100644 --- ql/src/test/results/clientpositive/join32_lessSize.q.out +++ ql/src/test/results/clientpositive/join32_lessSize.q.out @@ -108,22 +108,22 @@ TOK_QUERY STAGE DEPENDENCIES: - Stage-8 is a root stage + Stage-9 is a root stage + Stage-7 depends on stages: Stage-9 + Stage-8 depends on stages: Stage-7 Stage-6 depends on stages: Stage-8 - Stage-7 depends on stages: Stage-6 - Stage-5 depends on stages: Stage-7 - Stage-0 depends on stages: Stage-5 - Stage-2 depends on stages: Stage-0 + Stage-0 depends on stages: Stage-6 + Stage-3 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-8 + Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_1:$hdt$_2:x + $hdt$_2:x Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_1:$hdt$_2:x + $hdt$_2:x TableScan alias: x Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE @@ -142,7 +142,7 @@ STAGE PLANS: 1 _col1 (type: string) Position of Big Table: 0 - Stage: Stage-6 + Stage: Stage-7 Map Reduce Map Operator Tree: TableScan @@ -279,9 +279,9 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [$hdt$_1:$hdt$_1:z] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_1:z] - Stage: Stage-7 + Stage: Stage-8 Map Reduce Local Work Alias -> Map Local Tables: $hdt$_0:y @@ -303,11 +303,11 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - Position of Big Table: 1 + 0 _col3 (type: string) + 1 _col0 (type: string) + Position of Big Table: 0 - Stage: Stage-5 + Stage: Stage-6 Map Reduce Map Operator Tree: TableScan @@ -316,13 +316,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - outputColumnNames: _col1, _col2, _col5 - Position of Big Table: 1 + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 + Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col5 (type: string), _col2 (type: string), _col1 (type: string) + expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -445,7 +445,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 - Stage: Stage-2 + Stage: Stage-3 Stats-Aggr Operator #### A masked pattern was here #### @@ -653,24 +653,24 @@ TOK_QUERY STAGE DEPENDENCIES: - Stage-11 is a root stage + Stage-13 is a root stage + Stage-10 depends on stages: Stage-13 + Stage-12 depends on stages: Stage-10 + Stage-9 depends on stages: Stage-12 + Stage-11 depends on stages: Stage-9 Stage-8 depends on stages: Stage-11 - Stage-10 depends on stages: Stage-8 - Stage-7 depends on stages: Stage-10 - Stage-9 depends on stages: Stage-7 - Stage-6 depends on stages: Stage-9 - Stage-0 depends on stages: Stage-6 - Stage-2 depends on stages: Stage-0 + Stage-0 depends on stages: Stage-8 + Stage-4 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-11 + Stage: Stage-13 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_1:$hdt$_2:$hdt$_2:x + $hdt$_2:x Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_1:$hdt$_2:$hdt$_2:x + $hdt$_2:x TableScan alias: x Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE @@ -689,7 +689,7 @@ STAGE PLANS: 1 _col0 (type: string) Position of Big Table: 1 - Stage: Stage-8 + Stage: Stage-10 Map Reduce Map Operator Tree: TableScan @@ -780,16 +780,16 @@ STAGE PLANS: name: default.src1 name: default.src1 Truncated Path -> Alias: - /src1 [$hdt$_1:$hdt$_2:$hdt$_3:x] + /src1 [$hdt$_3:x] - Stage: Stage-10 + Stage: Stage-12 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_1:$hdt$_1:w + $hdt$_1:w Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_1:$hdt$_1:w + $hdt$_1:w TableScan alias: w Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -804,11 +804,11 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - Position of Big Table: 1 + 0 _col1 (type: string) + 1 _col0 (type: string) + Position of Big Table: 0 - Stage: Stage-7 + Stage: Stage-9 Map Reduce Map Operator Tree: TableScan @@ -817,10 +817,10 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col4 - Position of Big Table: 1 + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3 + Position of Big Table: 0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -831,7 +831,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col1,_col4 + columns _col0,_col3 columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -846,7 +846,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: -mr-10002 + base file name: -mr-10001 input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: @@ -911,7 +911,7 @@ STAGE PLANS: Truncated Path -> Alias: #### A masked pattern was here #### - Stage: Stage-9 + Stage: Stage-11 Map Reduce Local Work Alias -> Map Local Tables: $hdt$_0:w @@ -934,10 +934,10 @@ STAGE PLANS: HashTable Sink Operator keys: 0 _col0 (type: string) - 1 _col1 (type: string) - Position of Big Table: 1 + 1 _col0 (type: string) + Position of Big Table: 0 - Stage: Stage-6 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan @@ -947,12 +947,12 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col3, _col6 - Position of Big Table: 1 + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 + Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col6 (type: string), _col1 (type: string) + expressions: _col0 (type: string), _col3 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -993,11 +993,11 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: -mr-10001 + base file name: -mr-10002 input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col1,_col4 + columns _col0,_col3 columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -1006,7 +1006,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col1,_col4 + columns _col0,_col3 columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -1085,7 +1085,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 - Stage: Stage-2 + Stage: Stage-4 Stats-Aggr Operator #### A masked pattern was here #### @@ -1109,7 +1109,7 @@ POSTHOOK: Input: default@src1 POSTHOOK: Output: default@dest_j1 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)w.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value EXPRESSION [(src1)x.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: select * from dest_j1 PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 diff --git ql/src/test/results/clientpositive/join33.q.out ql/src/test/results/clientpositive/join33.q.out index 49e302a..5795669 100644 --- ql/src/test/results/clientpositive/join33.q.out +++ ql/src/test/results/clientpositive/join33.q.out @@ -100,19 +100,19 @@ TOK_QUERY STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-5 depends on stages: Stage-7 - Stage-0 depends on stages: Stage-5 - Stage-2 depends on stages: Stage-0 + Stage-8 is a root stage + Stage-6 depends on stages: Stage-8 + Stage-0 depends on stages: Stage-6 + Stage-3 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-8 Map Reduce Local Work Alias -> Map Local Tables: $hdt$_0:y Fetch Operator limit: -1 - $hdt$_1:$hdt$_2:x + $hdt$_2:x Fetch Operator limit: -1 Alias -> Map Local Operator Tree: @@ -131,10 +131,10 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - Position of Big Table: 1 - $hdt$_1:$hdt$_2:x + 0 _col3 (type: string) + 1 _col0 (type: string) + Position of Big Table: 0 + $hdt$_2:x TableScan alias: x Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE @@ -153,7 +153,7 @@ STAGE PLANS: 1 _col1 (type: string) Position of Big Table: 0 - Stage: Stage-5 + Stage: Stage-6 Map Reduce Map Operator Tree: TableScan @@ -181,13 +181,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - outputColumnNames: _col1, _col2, _col5 - Position of Big Table: 1 + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 + Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col5 (type: string), _col2 (type: string), _col1 (type: string) + expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -356,7 +356,7 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [$hdt$_1:$hdt$_1:z] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_1:z] Stage: Stage-0 Move Operator @@ -380,7 +380,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_j1 - Stage: Stage-2 + Stage: Stage-3 Stats-Aggr Operator #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/join_alt_syntax.q.out ql/src/test/results/clientpositive/join_alt_syntax.q.out index cc908c1..d119ab5 100644 --- ql/src/test/results/clientpositive/join_alt_syntax.q.out +++ ql/src/test/results/clientpositive/join_alt_syntax.q.out @@ -359,13 +359,13 @@ where p2.p_name = p3.p_name and p1.p_partkey = p4.p_partkey and p1.p_partkey = p2.p_partkey POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-2 depends on stages: Stage-3 - Stage-1 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-1 + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 STAGE PLANS: - Stage: Stage-3 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan @@ -418,6 +418,13 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + key expressions: _col3 (type: string) + sort order: + + Map-reduce partition columns: _col3 (type: string) + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) + TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -432,21 +439,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - TableScan - Reduce Output Operator - key expressions: _col3 (type: string) - sort order: + - Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col4 + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -455,10 +455,17 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-1 + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string) + TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -474,24 +481,17 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - TableScan - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string), _col4 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col1, _col2, _col4, _col6 + 1 _col0 (type: int) + outputColumnNames: _col1, _col3, _col4, _col6 Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: string), _col6 (type: string), _col2 (type: string), _col1 (type: string) + expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -519,13 +519,13 @@ where p2.p_name = p3.p_name and p1.p_partkey = p4.p_partkey and p1.p_partkey = p2.p_partkey POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-2 depends on stages: Stage-3 - Stage-1 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-1 + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 STAGE PLANS: - Stage: Stage-3 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan @@ -578,6 +578,13 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + key expressions: _col3 (type: string) + sort order: + + Map-reduce partition columns: _col3 (type: string) + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) + TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -592,21 +599,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - TableScan - Reduce Output Operator - key expressions: _col3 (type: string) - sort order: + - Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col4 + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -615,10 +615,17 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-1 + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string) + TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -634,24 +641,17 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - TableScan - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string), _col4 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col1, _col2, _col4, _col6 + 1 _col0 (type: int) + outputColumnNames: _col1, _col3, _col4, _col6 Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: string), _col6 (type: string), _col2 (type: string), _col1 (type: string) + expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out index b6e5b50..d565c7f 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out @@ -281,7 +281,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[15][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 on p2.p_partkey = 1 and p3.p_name = p2.p_name PREHOOK: type: QUERY @@ -289,12 +289,12 @@ POSTHOOK: query: explain select * from part p1 join part p2 join part p3 on p2.p_partkey = 1 and p3.p_name = p2.p_name POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-1 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-1 + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: - Stage: Stage-2 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan @@ -338,21 +338,22 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-1 + Stage: Stage-2 Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -363,11 +364,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - TableScan - Reduce Output Operator - sort order: - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -375,10 +371,10 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), 1 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + expressions: _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), 1 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/join_cond_pushdown_2.q.out ql/src/test/results/clientpositive/join_cond_pushdown_2.q.out index 98008ad..55c37e4 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_2.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_2.q.out @@ -141,13 +141,13 @@ from part p1 join part p2 join part p3 on p2.p_name = p1.p_name join part p4 on and p1.p_partkey = p2.p_partkey POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-2 depends on stages: Stage-3 - Stage-1 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-1 + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 STAGE PLANS: - Stage: Stage-3 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan @@ -202,6 +202,13 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + key expressions: _col10 (type: string) + sort order: + + Map-reduce partition columns: _col10 (type: string) + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -217,20 +224,13 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - TableScan - Reduce Output Operator - key expressions: _col10 (type: string) - sort order: + - Map-reduce partition columns: _col10 (type: string) - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col10 (type: string) + 0 _col10 (type: string) + 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -240,10 +240,17 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-1 + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string) + TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -259,33 +266,22 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - TableScan - Reduce Output Operator - key expressions: _col9 (type: int) - sort order: + - Map-reduce partition columns: _col9 (type: int) - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col9 (type: int) + 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: string), _col30 (type: string), _col31 (type: string), _col32 (type: int), _col33 (type: string), _col34 (type: double), _col35 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 + File Output Operator + compressed: false Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out index c098105..9077fc0 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out @@ -287,7 +287,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[15][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 where p2.p_partkey = 1 and p3.p_name = p2.p_name @@ -297,12 +297,12 @@ from part p1 join part p2 join part p3 where p2.p_partkey = 1 and p3.p_name = p2.p_name POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-1 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-1 + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: - Stage: Stage-2 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan @@ -346,21 +346,22 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-1 + Stage: Stage-2 Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -371,11 +372,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - TableScan - Reduce Output Operator - sort order: - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -383,10 +379,10 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), 1 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + expressions: _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), 1 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/join_cond_pushdown_4.q.out ql/src/test/results/clientpositive/join_cond_pushdown_4.q.out index a1dd24e..55e7678 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_4.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_4.q.out @@ -145,13 +145,13 @@ where p2.p_name = p3.p_name and p1.p_partkey = p4.p_partkey and p1.p_partkey = p2.p_partkey POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-2 depends on stages: Stage-3 - Stage-1 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-1 + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 STAGE PLANS: - Stage: Stage-3 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan @@ -206,6 +206,13 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + key expressions: _col10 (type: string) + sort order: + + Map-reduce partition columns: _col10 (type: string) + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -221,20 +228,13 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - TableScan - Reduce Output Operator - key expressions: _col10 (type: string) - sort order: + - Map-reduce partition columns: _col10 (type: string) - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col10 (type: string) + 0 _col10 (type: string) + 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -244,10 +244,17 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-1 + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string) + TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -263,33 +270,22 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - TableScan - Reduce Output Operator - key expressions: _col9 (type: int) - sort order: + - Map-reduce partition columns: _col9 (type: int) - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: int) - 1 _col9 (type: int) + 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: string), _col30 (type: string), _col31 (type: string), _col32 (type: int), _col33 (type: string), _col34 (type: double), _col35 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 + File Output Operator + compressed: false Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out index 3fba77c..9775f30 100644 --- ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out +++ ql/src/test/results/clientpositive/mapjoin_mapjoin.q.out @@ -56,18 +56,18 @@ TOK_QUERY STAGE DEPENDENCIES: - Stage-6 is a root stage - Stage-4 depends on stages: Stage-6 - Stage-0 depends on stages: Stage-4 + Stage-7 is a root stage + Stage-5 depends on stages: Stage-7 + Stage-0 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-6 + Stage: Stage-7 Map Reduce Local Work Alias -> Map Local Tables: $hdt$_0:src Fetch Operator limit: -1 - $hdt$_1:$hdt$_2:src1 + $hdt$_2:src1 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: @@ -86,10 +86,10 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - Position of Big Table: 1 - $hdt$_1:$hdt$_2:src1 + 0 _col1 (type: string) + 1 _col0 (type: string) + Position of Big Table: 0 + $hdt$_2:src1 TableScan alias: src1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE @@ -108,7 +108,7 @@ STAGE PLANS: 1 _col0 (type: string) Position of Big Table: 0 - Stage: Stage-4 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -136,36 +136,32 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 - Position of Big Table: 1 + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Position of Big Table: 0 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0 - columns.types string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -444,10 +440,10 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [$hdt$_1:$hdt$_1:srcpart] - /srcpart/ds=2008-04-08/hr=12 [$hdt$_1:$hdt$_1:srcpart] - /srcpart/ds=2008-04-09/hr=11 [$hdt$_1:$hdt$_1:srcpart] - /srcpart/ds=2008-04-09/hr=12 [$hdt$_1:$hdt$_1:srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_1:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_1:srcpart] + /srcpart/ds=2008-04-09/hr=11 [$hdt$_1:srcpart] + /srcpart/ds=2008-04-09/hr=12 [$hdt$_1:srcpart] Stage: Stage-0 Fetch Operator @@ -462,18 +458,18 @@ POSTHOOK: query: explain select srcpart.key from srcpart join src on (srcpart.value=src.value) join src1 on (srcpart.key=src1.key) where srcpart.value > 'val_450' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-6 is a root stage - Stage-4 depends on stages: Stage-6 - Stage-0 depends on stages: Stage-4 + Stage-7 is a root stage + Stage-5 depends on stages: Stage-7 + Stage-0 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-6 + Stage: Stage-7 Map Reduce Local Work Alias -> Map Local Tables: $hdt$_0:src Fetch Operator limit: -1 - $hdt$_1:$hdt$_2:src1 + $hdt$_2:src1 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: @@ -490,9 +486,9 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - $hdt$_1:$hdt$_2:src1 + 0 _col1 (type: string) + 1 _col0 (type: string) + $hdt$_2:src1 TableScan alias: src1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE @@ -508,7 +504,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) - Stage: Stage-4 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -533,21 +529,17 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col0 + File Output Operator + compressed: false Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/spark/auto_join13.q.out ql/src/test/results/clientpositive/spark/auto_join13.q.out index c81d296..0ffe9e3 100644 --- ql/src/test/results/clientpositive/spark/auto_join13.q.out +++ ql/src/test/results/clientpositive/spark/auto_join13.q.out @@ -30,22 +30,22 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 1 + Map 3 Map Operator Tree: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(key) < 200.0) and UDFToDouble(key) is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (UDFToDouble(key) < 100.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 UDFToDouble(_col0) (type: double) - 1 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) + 0 _col0 (type: string) + 1 _col0 (type: string) Local Work: Map Reduce Local Work Map 4 @@ -54,26 +54,26 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (UDFToDouble(key) < 100.0) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + predicate: ((UDFToDouble(key) < 200.0) and UDFToDouble(key) is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + 0 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) + 1 UDFToDouble(_col0) (type: double) Local Work: Map Reduce Local Work Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 1) #### A masked pattern was here #### Vertices: - Map 2 + Map 1 Map Operator Tree: TableScan alias: src @@ -93,7 +93,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 input vertices: - 1 Map 4 + 1 Map 3 Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (UDFToDouble(_col2) + UDFToDouble(_col0)) is not null (type: boolean) @@ -102,14 +102,14 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 UDFToDouble(_col0) (type: double) - 1 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) - outputColumnNames: _col2, _col3 + 0 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) + 1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col1, _col2 input vertices: - 0 Map 1 + 1 Map 4 Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: hash(_col3,_col2) (type: int) + expressions: hash(_col2,_col1) (type: int) outputColumnNames: _col0 Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -123,7 +123,7 @@ STAGE PLANS: value expressions: _col0 (type: bigint) Local Work: Map Reduce Local Work - Reducer 3 + Reducer 2 Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) diff --git ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out index 72f60d0..a234ff5 100644 --- ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out +++ ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out @@ -324,9 +324,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 2), Map 6 (PARTITION-LEVEL SORT, 2) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2) - Reducer 3 <- Reducer 2 (SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Reducer 3 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -335,23 +335,6 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator predicate: ((UDFToDouble(key) > 100.0) and value is not null) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -364,7 +347,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Map 6 + Map 5 Map Operator Tree: TableScan alias: a @@ -381,6 +364,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Map 6 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -388,18 +388,30 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col2 + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 3 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Reducer 3 + Reducer 4 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) @@ -415,22 +427,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/join13.q.out ql/src/test/results/clientpositive/spark/join13.q.out index 3eb7720..4a045f7 100644 --- ql/src/test/results/clientpositive/spark/join13.q.out +++ ql/src/test/results/clientpositive/spark/join13.q.out @@ -32,8 +32,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -42,23 +42,6 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(key) < 200.0) and UDFToDouble(key) is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Map 3 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator predicate: (UDFToDouble(key) < 100.0) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -71,7 +54,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: src @@ -88,33 +71,29 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 200.0) and UDFToDouble(key) is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 UDFToDouble(_col0) (type: double) - 1 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) - outputColumnNames: _col2, _col3 - Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col3 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2 @@ -128,6 +107,27 @@ STAGE PLANS: Map-reduce partition columns: (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string) + Reducer 3 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 (UDFToDouble(_col2) + UDFToDouble(_col0)) (type: double) + 1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 1065 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/join32.q.out ql/src/test/results/clientpositive/spark/join32.q.out index d7383fb..4ae9dc6 100644 --- ql/src/test/results/clientpositive/spark/join32.q.out +++ ql/src/test/results/clientpositive/spark/join32.q.out @@ -110,25 +110,25 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 1 + Map 2 Map Operator Tree: TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: key is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 _col0 (type: string) - 1 _col3 (type: string) - Position of Big Table: 1 + 1 _col1 (type: string) + Position of Big Table: 0 Local Work: Map Reduce Local Work Path -> Alias: @@ -136,7 +136,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: src + base file name: src1 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -146,14 +146,14 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -166,38 +166,38 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src + name: default.src1 + name: default.src1 Truncated Path -> Alias: - /src [y] + /src1 [x] Map 3 Map Operator Tree: TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value is not null and key is not null) (type: boolean) - Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: string) - 1 _col1 (type: string) + 0 _col3 (type: string) + 1 _col0 (type: string) Position of Big Table: 0 Local Work: Map Reduce Local Work @@ -206,7 +206,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: src1 + base file name: src input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -216,14 +216,14 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -236,26 +236,26 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src1 - name: default.src1 + name: default.src + name: default.src Truncated Path -> Alias: - /src1 [x] + /src [y] Stage: Stage-1 Spark #### A masked pattern was here #### Vertices: - Map 2 + Map 1 Map Operator Tree: TableScan alias: z @@ -277,22 +277,22 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Map 2 Position of Big Table: 0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - outputColumnNames: _col1, _col2, _col5 + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 input vertices: - 0 Map 1 - Position of Big Table: 1 + 1 Map 3 + Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col5 (type: string), _col2 (type: string), _col1 (type: string) + expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/spark/join32_lessSize.q.out ql/src/test/results/clientpositive/spark/join32_lessSize.q.out index b1efc8f..78bb655 100644 --- ql/src/test/results/clientpositive/spark/join32_lessSize.q.out +++ ql/src/test/results/clientpositive/spark/join32_lessSize.q.out @@ -118,25 +118,25 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 1 + Map 2 Map Operator Tree: TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: key is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 _col0 (type: string) - 1 _col3 (type: string) - Position of Big Table: 1 + 1 _col1 (type: string) + Position of Big Table: 0 Local Work: Map Reduce Local Work Path -> Alias: @@ -144,7 +144,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: src + base file name: src1 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -154,14 +154,14 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -174,38 +174,38 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src + name: default.src1 + name: default.src1 Truncated Path -> Alias: - /src [y] + /src1 [x] Map 3 Map Operator Tree: TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value is not null and key is not null) (type: boolean) - Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: string) - 1 _col1 (type: string) + 0 _col3 (type: string) + 1 _col0 (type: string) Position of Big Table: 0 Local Work: Map Reduce Local Work @@ -214,7 +214,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: src1 + base file name: src input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -224,14 +224,14 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -244,26 +244,26 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src1 - name: default.src1 + name: default.src + name: default.src Truncated Path -> Alias: - /src1 [x] + /src [y] Stage: Stage-1 Spark #### A masked pattern was here #### Vertices: - Map 2 + Map 1 Map Operator Tree: TableScan alias: z @@ -285,22 +285,22 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Map 2 Position of Big Table: 0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - outputColumnNames: _col1, _col2, _col5 + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 input vertices: - 0 Map 1 - Position of Big Table: 1 + 1 Map 3 + Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col5 (type: string), _col2 (type: string), _col1 (type: string) + expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -624,7 +624,7 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 3 + Map 1 Map Operator Tree: TableScan alias: x @@ -699,25 +699,36 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 1 + Map 2 Map Operator Tree: TableScan - alias: w - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 keys: 0 _col0 (type: string) - 1 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col3 + input vertices: + 0 Map 1 Position of Big Table: 1 + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + Position of Big Table: 1 Local Work: Map Reduce Local Work Path -> Alias: @@ -725,7 +736,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: src + base file name: src1 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -735,14 +746,14 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -755,50 +766,39 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src + name: default.src1 + name: default.src1 Truncated Path -> Alias: - /src [w] + /src1 [x] Map 4 Map Operator Tree: TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + alias: w + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - input vertices: - 0 Map 3 - Position of Big Table: 1 - Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - Position of Big Table: 0 + Position of Big Table: 0 Local Work: Map Reduce Local Work Path -> Alias: @@ -806,7 +806,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: src1 + base file name: src input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -816,14 +816,14 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -836,26 +836,26 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src1 - name: default.src1 + name: default.src + name: default.src Truncated Path -> Alias: - /src1 [x] + /src [w] Stage: Stage-1 Spark #### A masked pattern was here #### Vertices: - Map 2 + Map 3 Map Operator Tree: TableScan alias: w @@ -873,26 +873,26 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col4 + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3 input vertices: - 1 Map 4 - Position of Big Table: 0 + 0 Map 2 + Position of Big Table: 1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1, _col3, _col6 + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 input vertices: - 0 Map 1 - Position of Big Table: 1 + 1 Map 4 + Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col6 (type: string), _col1 (type: string) + expressions: _col0 (type: string), _col3 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1029,7 +1029,7 @@ POSTHOOK: Input: default@src1 POSTHOOK: Output: default@dest_j1 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)w.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest_j1.value EXPRESSION [(src1)x.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: select * from dest_j1 PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 diff --git ql/src/test/results/clientpositive/spark/join33.q.out ql/src/test/results/clientpositive/spark/join33.q.out index d7383fb..4ae9dc6 100644 --- ql/src/test/results/clientpositive/spark/join33.q.out +++ ql/src/test/results/clientpositive/spark/join33.q.out @@ -110,25 +110,25 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 1 + Map 2 Map Operator Tree: TableScan - alias: y - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: key is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 _col0 (type: string) - 1 _col3 (type: string) - Position of Big Table: 1 + 1 _col1 (type: string) + Position of Big Table: 0 Local Work: Map Reduce Local Work Path -> Alias: @@ -136,7 +136,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: src + base file name: src1 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -146,14 +146,14 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -166,38 +166,38 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src + name: default.src1 + name: default.src1 Truncated Path -> Alias: - /src [y] + /src1 [x] Map 3 Map Operator Tree: TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value is not null and key is not null) (type: boolean) - Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: string) - 1 _col1 (type: string) + 0 _col3 (type: string) + 1 _col0 (type: string) Position of Big Table: 0 Local Work: Map Reduce Local Work @@ -206,7 +206,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: src1 + base file name: src input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -216,14 +216,14 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -236,26 +236,26 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src1 - name: default.src1 + name: default.src + name: default.src Truncated Path -> Alias: - /src1 [x] + /src [y] Stage: Stage-1 Spark #### A masked pattern was here #### Vertices: - Map 2 + Map 1 Map Operator Tree: TableScan alias: z @@ -277,22 +277,22 @@ STAGE PLANS: 1 _col1 (type: string) outputColumnNames: _col0, _col3 input vertices: - 1 Map 3 + 1 Map 2 Position of Big Table: 0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - outputColumnNames: _col1, _col2, _col5 + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col3, _col6 input vertices: - 0 Map 1 - Position of Big Table: 1 + 1 Map 3 + Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col5 (type: string), _col2 (type: string), _col1 (type: string) + expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out index f34153d..a68321f 100644 --- ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out +++ ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out @@ -394,9 +394,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 2), Map 7 (PARTITION-LEVEL SORT, 2) - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 2), Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -405,35 +405,17 @@ STAGE PLANS: alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + predicate: (p_partkey is not null and p_name is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_partkey (type: int), p_name (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Map 3 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan @@ -451,76 +433,94 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + Map 6 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Map 7 Map Operator Tree: TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (p_partkey is not null and p_name is not null) (type: boolean) - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_partkey (type: int), p_name (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col1, _col2, _col4, _col6 - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col4 (type: string), _col6 (type: string), _col2 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + 0 _col0 (type: int), _col1 (type: string) + 1 _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string) + sort order: + + Map-reduce partition columns: _col3 (type: string) + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) + Reducer 3 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col4 + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: int) + key expressions: _col0 (type: int) sort order: + - Map-reduce partition columns: _col1 (type: int) + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string), _col4 (type: string) - Reducer 6 + value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string) + Reducer 4 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: int), _col1 (type: string) - 1 _col0 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: string) - sort order: + - Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col3, _col4, _col6 + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -546,9 +546,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 2), Map 7 (PARTITION-LEVEL SORT, 2) - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 2), Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -557,35 +557,17 @@ STAGE PLANS: alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_partkey (type: int), p_name (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Map 3 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: TableScan @@ -603,76 +585,94 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + Map 6 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Map 7 Map Operator Tree: TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (p_name is not null and p_partkey is not null) (type: boolean) - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_partkey (type: int), p_name (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: int) - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col1, _col2, _col4, _col6 - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col4 (type: string), _col6 (type: string), _col2 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + 0 _col1 (type: string), _col0 (type: int) + 1 _col1 (type: string), _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string) + sort order: + + Map-reduce partition columns: _col3 (type: string) + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) + Reducer 3 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col4 + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: int) + key expressions: _col0 (type: int) sort order: + - Map-reduce partition columns: _col1 (type: int) + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string), _col4 (type: string) - Reducer 6 + value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string) + Reducer 4 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string), _col0 (type: int) - 1 _col1 (type: string), _col0 (type: int) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: string) - sort order: + - Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col3, _col4, _col6 + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/join_cond_pushdown_1.q.out ql/src/test/results/clientpositive/spark/join_cond_pushdown_1.q.out index aa84423..fe2da0e 100644 --- ql/src/test/results/clientpositive/spark/join_cond_pushdown_1.q.out +++ ql/src/test/results/clientpositive/spark/join_cond_pushdown_1.q.out @@ -301,7 +301,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[15][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 on p2.p_partkey = 1 and p3.p_name = p2.p_name PREHOOK: type: QUERY @@ -316,8 +316,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -325,19 +325,6 @@ STAGE PLANS: TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Map 3 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: p_name is not null (type: boolean) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE @@ -351,7 +338,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: p1 @@ -369,18 +356,45 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: + 0 _col1 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + Reducer 3 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), 1 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + expressions: _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), 1 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -390,24 +404,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/join_cond_pushdown_2.q.out ql/src/test/results/clientpositive/spark/join_cond_pushdown_2.q.out index 26e05ac..5f572b0 100644 --- ql/src/test/results/clientpositive/spark/join_cond_pushdown_2.q.out +++ ql/src/test/results/clientpositive/spark/join_cond_pushdown_2.q.out @@ -150,9 +150,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 2), Map 7 (PARTITION-LEVEL SORT, 2) - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 2), Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -161,36 +161,18 @@ STAGE PLANS: alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Map 3 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + predicate: (p_partkey is not null and p_name is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Map 5 Map Operator Tree: TableScan @@ -209,77 +191,91 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Map 6 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Map 7 Map Operator Tree: TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (p_partkey is not null and p_name is not null) (type: boolean) - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col9 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: string), _col30 (type: string), _col31 (type: string), _col32 (type: int), _col33 (type: string), _col34 (type: double), _col35 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + 0 _col0 (type: int), _col1 (type: string) + 1 _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col10 (type: string) + sort order: + + Map-reduce partition columns: _col10 (type: string) + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + Reducer 3 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col10 (type: string) + 0 _col10 (type: string) + 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col9 (type: int) + key expressions: _col0 (type: int) sort order: + - Map-reduce partition columns: _col9 (type: int) + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string) - Reducer 6 + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string) + Reducer 4 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: int), _col1 (type: string) - 1 _col0 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col10 (type: string) - sort order: + - Map-reduce partition columns: _col10 (type: string) - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/join_cond_pushdown_3.q.out ql/src/test/results/clientpositive/spark/join_cond_pushdown_3.q.out index 4607309..c0bdd80 100644 --- ql/src/test/results/clientpositive/spark/join_cond_pushdown_3.q.out +++ ql/src/test/results/clientpositive/spark/join_cond_pushdown_3.q.out @@ -307,7 +307,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[15][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 where p2.p_partkey = 1 and p3.p_name = p2.p_name @@ -324,8 +324,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -333,19 +333,6 @@ STAGE PLANS: TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Map 3 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: p_name is not null (type: boolean) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE @@ -359,7 +346,7 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Map 5 + Map 4 Map Operator Tree: TableScan alias: p1 @@ -377,18 +364,45 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: + 0 _col1 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + Reducer 3 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), 1 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + expressions: _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), 1 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -398,24 +412,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/join_cond_pushdown_4.q.out ql/src/test/results/clientpositive/spark/join_cond_pushdown_4.q.out index c821fe4..ddc6812 100644 --- ql/src/test/results/clientpositive/spark/join_cond_pushdown_4.q.out +++ ql/src/test/results/clientpositive/spark/join_cond_pushdown_4.q.out @@ -154,9 +154,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 2), Map 7 (PARTITION-LEVEL SORT, 2) - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 5 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 2), Reducer 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -165,36 +165,18 @@ STAGE PLANS: alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Map 3 - Map Operator Tree: - TableScan - alias: p1 - Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + predicate: (p_partkey is not null and p_name is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Map 5 Map Operator Tree: TableScan @@ -213,77 +195,91 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: string) Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Map 6 + Map Operator Tree: + TableScan + alias: p1 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Map 7 Map Operator Tree: TableScan alias: p1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (p_partkey is not null and p_name is not null) (type: boolean) - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + predicate: p_partkey is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 2 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col9 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string), _col27 (type: int), _col28 (type: string), _col29 (type: string), _col30 (type: string), _col31 (type: string), _col32 (type: int), _col33 (type: string), _col34 (type: double), _col35 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + 0 _col0 (type: int), _col1 (type: string) + 1 _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col10 (type: string) + sort order: + + Map-reduce partition columns: _col10 (type: string) + Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + Reducer 3 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col10 (type: string) + 0 _col10 (type: string) + 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col9 (type: int) + key expressions: _col0 (type: int) sort order: + - Map-reduce partition columns: _col9 (type: int) + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string) - Reducer 6 + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string), _col18 (type: int), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: int), _col24 (type: string), _col25 (type: double), _col26 (type: string) + Reducer 4 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: int), _col1 (type: string) - 1 _col0 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17 - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col10 (type: string) - sort order: + - Map-reduce partition columns: _col10 (type: string) - Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 (type: int), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: int), _col15 (type: string), _col16 (type: double), _col17 (type: string) + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35 + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/spark/mapjoin_mapjoin.q.out ql/src/test/results/clientpositive/spark/mapjoin_mapjoin.q.out index 7089004..3b2c80e 100644 --- ql/src/test/results/clientpositive/spark/mapjoin_mapjoin.q.out +++ ql/src/test/results/clientpositive/spark/mapjoin_mapjoin.q.out @@ -65,25 +65,25 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 1 + Map 2 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: value is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string) + expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 _col0 (type: string) - 1 _col1 (type: string) - Position of Big Table: 1 + 1 _col0 (type: string) + Position of Big Table: 0 Local Work: Map Reduce Local Work Path -> Alias: @@ -91,7 +91,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: src + base file name: src1 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -101,14 +101,14 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -121,37 +121,37 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src + name: default.src1 + name: default.src1 Truncated Path -> Alias: - /src [src] + /src1 [src1] Map 3 Map Operator Tree: TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: key is not null (type: boolean) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) + expressions: value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) Position of Big Table: 0 Local Work: @@ -161,7 +161,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: src1 + base file name: src input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -171,14 +171,14 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -191,26 +191,26 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src1 - name: default.src1 + name: default.src + name: default.src Truncated Path -> Alias: - /src1 [src1] + /src [src] Stage: Stage-1 Spark #### A masked pattern was here #### Vertices: - Map 2 + Map 1 Map Operator Tree: TableScan alias: srcpart @@ -232,45 +232,41 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Map 2 Position of Big Table: 0 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 input vertices: - 0 Map 1 - Position of Big Table: 1 + 1 Map 3 + Position of Big Table: 0 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 1 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0 - columns.types string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Local Work: Map Reduce Local Work Path -> Alias: @@ -488,39 +484,39 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 1 + Map 2 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value > 'val_450') (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string) + expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 _col0 (type: string) - 1 _col1 (type: string) + 1 _col0 (type: string) Local Work: Map Reduce Local Work Map 3 Map Operator Tree: TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + predicate: (value > 'val_450') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) + expressions: value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) Local Work: Map Reduce Local Work @@ -529,7 +525,7 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 2 + Map 1 Map Operator Tree: TableScan alias: srcpart @@ -549,29 +545,25 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Map 2 Statistics: Num rows: 183 Data size: 1951 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 input vertices: - 0 Map 1 + 1 Map 3 Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col0 + File Output Operator + compressed: false Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/tez/explainuser_1.q.out ql/src/test/results/clientpositive/tez/explainuser_1.q.out index c1bfaff..2147984 100644 --- ql/src/test/results/clientpositive/tez/explainuser_1.q.out +++ ql/src/test/results/clientpositive/tez/explainuser_1.q.out @@ -6427,77 +6427,74 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 3 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Map 1 <- Map 3 (BROADCAST_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 Reducer 2 - File Output Operator [FS_20] + File Output Operator [FS_19] compressed:true Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Select Operator [SEL_19] - outputColumnNames:["_col0"] - Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE - Merge Join Operator [MERGEJOIN_30] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"1":"_col1 (type: string)","0":"_col0 (type: string)"} - | outputColumnNames:["_col1"] - | Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE - |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_15] - | key expressions:_col0 (type: string) - | Map-reduce partition columns:_col0 (type: string) - | sort order:+ - | Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_2] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_26] - | predicate:(value > 'val_450') (type: boolean) - | Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_0] - | alias:src - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - |<-Map 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_17] - key expressions:_col1 (type: string) - Map-reduce partition columns:_col1 (type: string) - sort order:+ - Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE - value expressions:_col0 (type: string) - Map Join Operator [MAPJOIN_29] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 3":"_col0 (type: string)","Map 4":"_col0 (type: string)"} - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE - |<-Map 4 [BROADCAST_EDGE] - | Reduce Output Operator [RS_11] - | key expressions:_col0 (type: string) - | Map-reduce partition columns:_col0 (type: string) - | sort order:+ - | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_7] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_28] - | predicate:key is not null (type: boolean) - | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_6] - | alias:src1 - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE - |<-Select Operator [SEL_5] - outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_27] - predicate:(((value > 'val_450') and key is not null) and value is not null) (type: boolean) - Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_3] - alias:srcpart - Statistics:Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Merge Join Operator [MERGEJOIN_29] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"1":"_col0 (type: string)","0":"_col1 (type: string)"} + | outputColumnNames:["_col0"] + | Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE + |<-Map 1 [SIMPLE_EDGE] + | Reduce Output Operator [RS_14] + | key expressions:_col1 (type: string) + | Map-reduce partition columns:_col1 (type: string) + | sort order:+ + | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE + | value expressions:_col0 (type: string) + | Map Join Operator [MAPJOIN_28] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Map 1":"_col0 (type: string)","Map 3":"_col0 (type: string)"} + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE + | |<-Map 3 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_11] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) + | | sort order:+ + | | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + | | Select Operator [SEL_4] + | | outputColumnNames:["_col0"] + | | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + | | Filter Operator [FIL_26] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + | | TableScan [TS_3] + | | alias:src1 + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + | |<-Select Operator [SEL_2] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE + | Filter Operator [FIL_25] + | predicate:(((value > 'val_450') and key is not null) and value is not null) (type: boolean) + | Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE + | TableScan [TS_0] + | alias:srcpart + | Statistics:Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + |<-Map 4 [SIMPLE_EDGE] + Reduce Output Operator [RS_16] + key expressions:_col0 (type: string) + Map-reduce partition columns:_col0 (type: string) + sort order:+ + Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator [SEL_7] + outputColumnNames:["_col0"] + Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator [FIL_27] + predicate:(value > 'val_450') (type: boolean) + Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + TableScan [TS_5] + alias:src + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE PREHOOK: query: explain select srcpart.key from srcpart join src on (srcpart.value=src.value) join src1 on (srcpart.key=src1.key) where srcpart.value > 'val_450' PREHOOK: type: QUERY @@ -6506,77 +6503,74 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 3 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Map 1 <- Map 3 (BROADCAST_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 Reducer 2 - File Output Operator [FS_20] + File Output Operator [FS_19] compressed:true Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Select Operator [SEL_19] - outputColumnNames:["_col0"] - Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE - Merge Join Operator [MERGEJOIN_30] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"1":"_col1 (type: string)","0":"_col0 (type: string)"} - | outputColumnNames:["_col1"] - | Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE - |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_15] - | key expressions:_col0 (type: string) - | Map-reduce partition columns:_col0 (type: string) - | sort order:+ - | Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_2] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_26] - | predicate:(value > 'val_450') (type: boolean) - | Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_0] - | alias:src - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - |<-Map 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_17] - key expressions:_col1 (type: string) - Map-reduce partition columns:_col1 (type: string) - sort order:+ - Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE - value expressions:_col0 (type: string) - Map Join Operator [MAPJOIN_29] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 3":"_col0 (type: string)","Map 4":"_col0 (type: string)"} - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE - |<-Map 4 [BROADCAST_EDGE] - | Reduce Output Operator [RS_11] - | key expressions:_col0 (type: string) - | Map-reduce partition columns:_col0 (type: string) - | sort order:+ - | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_7] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_28] - | predicate:key is not null (type: boolean) - | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_6] - | alias:src1 - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE - |<-Select Operator [SEL_5] - outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_27] - predicate:(((value > 'val_450') and key is not null) and value is not null) (type: boolean) - Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_3] - alias:srcpart - Statistics:Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Merge Join Operator [MERGEJOIN_29] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"1":"_col0 (type: string)","0":"_col1 (type: string)"} + | outputColumnNames:["_col0"] + | Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE + |<-Map 1 [SIMPLE_EDGE] + | Reduce Output Operator [RS_14] + | key expressions:_col1 (type: string) + | Map-reduce partition columns:_col1 (type: string) + | sort order:+ + | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE + | value expressions:_col0 (type: string) + | Map Join Operator [MAPJOIN_28] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Map 1":"_col0 (type: string)","Map 3":"_col0 (type: string)"} + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE + | |<-Map 3 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_11] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) + | | sort order:+ + | | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + | | Select Operator [SEL_4] + | | outputColumnNames:["_col0"] + | | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + | | Filter Operator [FIL_26] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + | | TableScan [TS_3] + | | alias:src1 + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + | |<-Select Operator [SEL_2] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE + | Filter Operator [FIL_25] + | predicate:(((value > 'val_450') and key is not null) and value is not null) (type: boolean) + | Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE + | TableScan [TS_0] + | alias:srcpart + | Statistics:Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + |<-Map 4 [SIMPLE_EDGE] + Reduce Output Operator [RS_16] + key expressions:_col0 (type: string) + Map-reduce partition columns:_col0 (type: string) + sort order:+ + Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator [SEL_7] + outputColumnNames:["_col0"] + Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator [FIL_27] + predicate:(value > 'val_450') (type: boolean) + Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + TableScan [TS_5] + alias:src + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE PREHOOK: query: explain select srcpart.key from srcpart join src on (srcpart.value=src.value) join src1 on (srcpart.key=src1.key) where srcpart.value > 'val_450' PREHOOK: type: QUERY @@ -6585,77 +6579,74 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 3 <- Map 4 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +Map 1 <- Map 3 (BROADCAST_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 Reducer 2 - File Output Operator [FS_20] + File Output Operator [FS_19] compressed:true Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Select Operator [SEL_19] - outputColumnNames:["_col0"] - Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE - Merge Join Operator [MERGEJOIN_30] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"1":"_col1 (type: string)","0":"_col0 (type: string)"} - | outputColumnNames:["_col1"] - | Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE - |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_15] - | key expressions:_col0 (type: string) - | Map-reduce partition columns:_col0 (type: string) - | sort order:+ - | Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_2] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_26] - | predicate:(value > 'val_450') (type: boolean) - | Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_0] - | alias:src - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - |<-Map 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_17] - key expressions:_col1 (type: string) - Map-reduce partition columns:_col1 (type: string) - sort order:+ - Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE - value expressions:_col0 (type: string) - Map Join Operator [MAPJOIN_29] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 3":"_col0 (type: string)","Map 4":"_col0 (type: string)"} - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE - |<-Map 4 [BROADCAST_EDGE] - | Reduce Output Operator [RS_11] - | key expressions:_col0 (type: string) - | Map-reduce partition columns:_col0 (type: string) - | sort order:+ - | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_7] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_28] - | predicate:key is not null (type: boolean) - | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_6] - | alias:src1 - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE - |<-Select Operator [SEL_5] - outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_27] - predicate:(((value > 'val_450') and key is not null) and value is not null) (type: boolean) - Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_3] - alias:srcpart - Statistics:Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Merge Join Operator [MERGEJOIN_29] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"1":"_col0 (type: string)","0":"_col1 (type: string)"} + | outputColumnNames:["_col0"] + | Statistics:Num rows: 555 Data size: 48285 Basic stats: COMPLETE Column stats: COMPLETE + |<-Map 1 [SIMPLE_EDGE] + | Reduce Output Operator [RS_14] + | key expressions:_col1 (type: string) + | Map-reduce partition columns:_col1 (type: string) + | sort order:+ + | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE + | value expressions:_col0 (type: string) + | Map Join Operator [MAPJOIN_28] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Map 1":"_col0 (type: string)","Map 3":"_col0 (type: string)"} + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE + | |<-Map 3 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_11] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) + | | sort order:+ + | | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + | | Select Operator [SEL_4] + | | outputColumnNames:["_col0"] + | | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + | | Filter Operator [FIL_26] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + | | TableScan [TS_3] + | | alias:src1 + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + | |<-Select Operator [SEL_2] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE + | Filter Operator [FIL_25] + | predicate:(((value > 'val_450') and key is not null) and value is not null) (type: boolean) + | Statistics:Num rows: 666 Data size: 118548 Basic stats: COMPLETE Column stats: COMPLETE + | TableScan [TS_0] + | alias:srcpart + | Statistics:Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + |<-Map 4 [SIMPLE_EDGE] + Reduce Output Operator [RS_16] + key expressions:_col0 (type: string) + Map-reduce partition columns:_col0 (type: string) + sort order:+ + Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator [SEL_7] + outputColumnNames:["_col0"] + Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator [FIL_27] + predicate:(value > 'val_450') (type: boolean) + Statistics:Num rows: 166 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + TableScan [TS_5] + alias:src + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE PREHOOK: query: explain select p_mfgr, p_name, p_size, diff --git ql/src/test/results/clientpositive/tez/explainuser_2.q.out ql/src/test/results/clientpositive/tez/explainuser_2.q.out index c063421..71d8f41 100644 --- ql/src/test/results/clientpositive/tez/explainuser_2.q.out +++ ql/src/test/results/clientpositive/tez/explainuser_2.q.out @@ -181,83 +181,83 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 - File Output Operator [FS_19] + Reducer 3 + File Output Operator [FS_18] compressed:false Statistics:Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Select Operator [SEL_18] + Select Operator [SEL_17] outputColumnNames:["_col0","_col1","_col2"] Statistics:Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_29] + Merge Join Operator [MERGEJOIN_28] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"1":"_col3 (type: string)","0":"_col0 (type: string)"} - | outputColumnNames:["_col1","_col2","_col5"] + | keys:{"1":"_col0 (type: string)","0":"_col3 (type: string)"} + | outputColumnNames:["_col0","_col3","_col6"] | Statistics:Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE - |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_14] + |<-Map 5 [SIMPLE_EDGE] + | Reduce Output Operator [RS_15] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE | value expressions:_col1 (type: string) - | Select Operator [SEL_1] + | Select Operator [SEL_6] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_25] + | Filter Operator [FIL_26] | predicate:key is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_0] + | TableScan [TS_5] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 4 [SIMPLE_EDGE] - Reduce Output Operator [RS_16] + |<-Reducer 2 [SIMPLE_EDGE] + Reduce Output Operator [RS_13] key expressions:_col3 (type: string) Map-reduce partition columns:_col3 (type: string) sort order:+ Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE value expressions:_col0 (type: string) - Merge Join Operator [MERGEJOIN_28] + Merge Join Operator [MERGEJOIN_27] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{"1":"_col1 (type: string)","0":"_col0 (type: string)"} | outputColumnNames:["_col0","_col3"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Map 3 [SIMPLE_EDGE] + |<-Map 1 [SIMPLE_EDGE] | Reduce Output Operator [RS_8] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_4] + | Select Operator [SEL_2] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_26] + | Filter Operator [FIL_24] | predicate:value is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_2] + | TableScan [TS_0] | alias:z | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 5 [SIMPLE_EDGE] + |<-Map 4 [SIMPLE_EDGE] Reduce Output Operator [RS_10] key expressions:_col1 (type: string) Map-reduce partition columns:_col1 (type: string) sort order:+ Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE value expressions:_col0 (type: string) - Select Operator [SEL_6] + Select Operator [SEL_4] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_27] + Filter Operator [FIL_25] predicate:(value is not null and key is not null) (type: boolean) Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_5] + TableScan [TS_3] alias:x Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE @@ -320,260 +320,260 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 11 <- Map 10 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 14 (SIMPLE_EDGE) +Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) +Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) +Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 8 <- Map 7 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) -Reducer 9 <- Reducer 16 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 8 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) +Reducer 9 <- Map 13 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 Reducer 5 - File Output Operator [FS_71] + File Output Operator [FS_69] compressed:false Statistics:Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Limit [LIM_70] + Limit [LIM_68] Number of rows:100 Statistics:Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_69] + Select Operator [SEL_67] | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] | Statistics:Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE |<-Reducer 4 [SIMPLE_EDGE] - Reduce Output Operator [RS_68] + Reduce Output Operator [RS_66] key expressions:_col0 (type: string), _col1 (type: string), _col2 (type: string) sort order:+++ Statistics:Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE value expressions:_col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint) - Group By Operator [GBY_66] + Group By Operator [GBY_64] | aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"] | keys:KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] | Statistics:Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE |<-Reducer 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_65] + Reduce Output Operator [RS_63] key expressions:_col0 (type: string), _col1 (type: string), _col2 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string), _col2 (type: string) sort order:+++ Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE value expressions:_col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint) - Group By Operator [GBY_64] + Group By Operator [GBY_62] aggregations:["count(_col3)","count(_col4)","count(_col5)"] keys:_col0 (type: string), _col1 (type: string), _col2 (type: string) outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_62] + Select Operator [SEL_60] outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_113] + Merge Join Operator [MERGEJOIN_111] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{"1":"_col15 (type: string), _col17 (type: string)","0":"_col1 (type: string), _col3 (type: string)"} | outputColumnNames:["_col2","_col3","_col12","_col13","_col20","_col21"] | Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 2 [SIMPLE_EDGE] + |<-Reducer 11 [SIMPLE_EDGE] | Reduce Output Operator [RS_58] - | key expressions:_col1 (type: string), _col3 (type: string) - | Map-reduce partition columns:_col1 (type: string), _col3 (type: string) + | key expressions:_col15 (type: string), _col17 (type: string) + | Map-reduce partition columns:_col15 (type: string), _col17 (type: string) | sort order:++ - | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE - | value expressions:_col2 (type: string) - | Merge Join Operator [MERGEJOIN_107] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"} - | | outputColumnNames:["_col1","_col2","_col3"] - | | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE - | |<-Map 1 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_53] - | | key expressions:_col0 (type: string) - | | Map-reduce partition columns:_col0 (type: string) - | | sort order:+ - | | Statistics:Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col1 (type: string), _col2 (type: string), _col3 (type: string) - | | Select Operator [SEL_1] - | | outputColumnNames:["_col0","_col1","_col2","_col3"] - | | Statistics:Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_99] - | | predicate:((k1 is not null and v2 is not null) and v3 is not null) (type: boolean) - | | Statistics:Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_0] - | | alias:cs - | | Statistics:Num rows: 170 Data size: 5890 Basic stats: COMPLETE Column stats: NONE - | |<-Map 6 [SIMPLE_EDGE] - | Reduce Output Operator [RS_55] - | key expressions:_col0 (type: string) - | Map-reduce partition columns:_col0 (type: string) - | sort order:+ - | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_4] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_100] - | predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) - | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_2] - | alias:d1 - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 9 [SIMPLE_EDGE] - Reduce Output Operator [RS_60] - key expressions:_col15 (type: string), _col17 (type: string) - Map-reduce partition columns:_col15 (type: string), _col17 (type: string) + | Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col6 (type: string), _col7 (type: string), _col14 (type: string) + | Select Operator [SEL_49] + | outputColumnNames:["_col14","_col15","_col17","_col6","_col7"] + | Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + | Merge Join Operator [MERGEJOIN_110] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"1":"_col2 (type: string), _col4 (type: string)","0":"_col4 (type: string), _col6 (type: string)"} + | | outputColumnNames:["_col2","_col3","_col14","_col15","_col17"] + | | Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 10 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_45] + | | key expressions:_col4 (type: string), _col6 (type: string) + | | Map-reduce partition columns:_col4 (type: string), _col6 (type: string) + | | sort order:++ + | | Statistics:Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col2 (type: string), _col3 (type: string) + | | Merge Join Operator [MERGEJOIN_108] + | | | condition map:[{"":"Inner Join 0 to 1"}] + | | | keys:{"1":"_col1 (type: string)","0":"_col3 (type: string)"} + | | | outputColumnNames:["_col2","_col3","_col4","_col6"] + | | | Statistics:Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + | | |<-Map 14 [SIMPLE_EDGE] + | | | Reduce Output Operator [RS_42] + | | | key expressions:_col1 (type: string) + | | | Map-reduce partition columns:_col1 (type: string) + | | | sort order:+ + | | | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + | | | Select Operator [SEL_16] + | | | outputColumnNames:["_col1"] + | | | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + | | | Filter Operator [FIL_102] + | | | predicate:((key = 'src1key') and value is not null) (type: boolean) + | | | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + | | | TableScan [TS_14] + | | | alias:src1 + | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | | |<-Reducer 9 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_40] + | | key expressions:_col3 (type: string) + | | Map-reduce partition columns:_col3 (type: string) + | | sort order:+ + | | Statistics:Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col2 (type: string), _col4 (type: string), _col6 (type: string) + | | Merge Join Operator [MERGEJOIN_107] + | | | condition map:[{"":"Inner Join 0 to 1"}] + | | | keys:{"1":"_col0 (type: string)","0":"_col2 (type: string)"} + | | | outputColumnNames:["_col2","_col3","_col4","_col6"] + | | | Statistics:Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + | | |<-Map 13 [SIMPLE_EDGE] + | | | Reduce Output Operator [RS_37] + | | | key expressions:_col0 (type: string) + | | | Map-reduce partition columns:_col0 (type: string) + | | | sort order:+ + | | | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + | | | Select Operator [SEL_13] + | | | outputColumnNames:["_col0"] + | | | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + | | | Filter Operator [FIL_101] + | | | predicate:((value = 'd1value') and key is not null) (type: boolean) + | | | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + | | | TableScan [TS_11] + | | | alias:d1 + | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | | |<-Reducer 8 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_35] + | | key expressions:_col2 (type: string) + | | Map-reduce partition columns:_col2 (type: string) + | | sort order:+ + | | Statistics:Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col3 (type: string), _col4 (type: string), _col6 (type: string) + | | Merge Join Operator [MERGEJOIN_106] + | | | condition map:[{"":"Inner Join 0 to 1"}] + | | | keys:{"1":"_col3 (type: string)","0":"_col1 (type: string)"} + | | | outputColumnNames:["_col2","_col3","_col4","_col6"] + | | | Statistics:Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + | | |<-Map 12 [SIMPLE_EDGE] + | | | Reduce Output Operator [RS_32] + | | | key expressions:_col3 (type: string) + | | | Map-reduce partition columns:_col3 (type: string) + | | | sort order:+ + | | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + | | | value expressions:_col0 (type: string), _col1 (type: string), _col2 (type: string), _col4 (type: string) + | | | Select Operator [SEL_10] + | | | outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] + | | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + | | | Filter Operator [FIL_100] + | | | predicate:((((((v3 = 'ssv3') and v2 is not null) and k1 is not null) and v1 is not null) and k2 is not null) and k3 is not null) (type: boolean) + | | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + | | | TableScan [TS_8] + | | | alias:ss + | | | Statistics:Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + | | |<-Map 7 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_30] + | | key expressions:_col1 (type: string) + | | Map-reduce partition columns:_col1 (type: string) + | | sort order:+ + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_7] + | | outputColumnNames:["_col1"] + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_99] + | | predicate:((key = 'srcpartkey') and value is not null) (type: boolean) + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_5] + | | alias:srcpart + | | Statistics:Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 16 [SIMPLE_EDGE] + | Reduce Output Operator [RS_47] + | key expressions:_col2 (type: string), _col4 (type: string) + | Map-reduce partition columns:_col2 (type: string), _col4 (type: string) + | sort order:++ + | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col3 (type: string), _col5 (type: string) + | Merge Join Operator [MERGEJOIN_109] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"} + | | outputColumnNames:["_col2","_col3","_col4","_col5"] + | | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + | |<-Map 15 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_24] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) + | | sort order:+ + | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + | | Select Operator [SEL_19] + | | outputColumnNames:["_col0","_col2","_col3","_col4","_col5"] + | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_103] + | | predicate:((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) + | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_17] + | | alias:sr + | | Statistics:Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + | |<-Map 17 [SIMPLE_EDGE] + | Reduce Output Operator [RS_26] + | key expressions:_col0 (type: string) + | Map-reduce partition columns:_col0 (type: string) + | sort order:+ + | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_22] + | outputColumnNames:["_col0"] + | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_104] + | predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_20] + | alias:d1 + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 2 [SIMPLE_EDGE] + Reduce Output Operator [RS_56] + key expressions:_col1 (type: string), _col3 (type: string) + Map-reduce partition columns:_col1 (type: string), _col3 (type: string) sort order:++ - Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE - value expressions:_col6 (type: string), _col7 (type: string), _col14 (type: string) - Select Operator [SEL_51] - outputColumnNames:["_col14","_col15","_col17","_col6","_col7"] - Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_112] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"1":"_col2 (type: string), _col4 (type: string)","0":"_col8 (type: string), _col10 (type: string)"} - | outputColumnNames:["_col6","_col7","_col14","_col15","_col17"] - | Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 16 [SIMPLE_EDGE] - | Reduce Output Operator [RS_49] - | key expressions:_col2 (type: string), _col4 (type: string) - | Map-reduce partition columns:_col2 (type: string), _col4 (type: string) - | sort order:++ - | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE - | value expressions:_col3 (type: string), _col5 (type: string) - | Merge Join Operator [MERGEJOIN_111] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"} - | | outputColumnNames:["_col2","_col3","_col4","_col5"] - | | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE - | |<-Map 15 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_36] - | | key expressions:_col0 (type: string) - | | Map-reduce partition columns:_col0 (type: string) - | | sort order:+ - | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) - | | Select Operator [SEL_31] - | | outputColumnNames:["_col0","_col2","_col3","_col4","_col5"] - | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_105] - | | predicate:((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) - | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_29] - | | alias:sr - | | Statistics:Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE - | |<-Map 17 [SIMPLE_EDGE] - | Reduce Output Operator [RS_38] - | key expressions:_col0 (type: string) - | Map-reduce partition columns:_col0 (type: string) - | sort order:+ - | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_34] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_106] - | predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) - | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_32] - | alias:d1 - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 8 [SIMPLE_EDGE] - Reduce Output Operator [RS_47] - key expressions:_col8 (type: string), _col10 (type: string) - Map-reduce partition columns:_col8 (type: string), _col10 (type: string) - sort order:++ - Statistics:Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE - value expressions:_col6 (type: string), _col7 (type: string) - Merge Join Operator [MERGEJOIN_110] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"1":"_col5 (type: string)","0":"_col1 (type: string)"} - | outputColumnNames:["_col6","_col7","_col8","_col10"] - | Statistics:Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE - |<-Map 7 [SIMPLE_EDGE] - | Reduce Output Operator [RS_42] - | key expressions:_col1 (type: string) - | Map-reduce partition columns:_col1 (type: string) - | sort order:+ - | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_7] - | outputColumnNames:["_col1"] - | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_101] - | predicate:((key = 'src1key') and value is not null) (type: boolean) - | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_5] - | alias:src1 - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 11 [SIMPLE_EDGE] - Reduce Output Operator [RS_44] - key expressions:_col5 (type: string) - Map-reduce partition columns:_col5 (type: string) - sort order:+ - Statistics:Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - value expressions:_col4 (type: string), _col6 (type: string), _col8 (type: string) - Merge Join Operator [MERGEJOIN_109] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"1":"_col2 (type: string)","0":"_col0 (type: string)"} - | outputColumnNames:["_col4","_col5","_col6","_col8"] - | Statistics:Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - |<-Map 10 [SIMPLE_EDGE] - | Reduce Output Operator [RS_24] - | key expressions:_col0 (type: string) - | Map-reduce partition columns:_col0 (type: string) - | sort order:+ - | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_10] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_102] - | predicate:((value = 'd1value') and key is not null) (type: boolean) - | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_8] - | alias:d1 - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 13 [SIMPLE_EDGE] - Reduce Output Operator [RS_26] - key expressions:_col2 (type: string) - Map-reduce partition columns:_col2 (type: string) - sort order:+ - Statistics:Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - value expressions:_col3 (type: string), _col4 (type: string), _col6 (type: string) - Merge Join Operator [MERGEJOIN_108] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"1":"_col3 (type: string)","0":"_col1 (type: string)"} - | outputColumnNames:["_col2","_col3","_col4","_col6"] - | Statistics:Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - |<-Map 12 [SIMPLE_EDGE] - | Reduce Output Operator [RS_18] - | key expressions:_col1 (type: string) - | Map-reduce partition columns:_col1 (type: string) - | sort order:+ - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_13] - | outputColumnNames:["_col1"] - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_103] - | predicate:((key = 'srcpartkey') and value is not null) (type: boolean) - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_11] - | alias:srcpart - | Statistics:Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - |<-Map 14 [SIMPLE_EDGE] - Reduce Output Operator [RS_20] - key expressions:_col3 (type: string) - Map-reduce partition columns:_col3 (type: string) - sort order:+ - Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - value expressions:_col0 (type: string), _col1 (type: string), _col2 (type: string), _col4 (type: string) - Select Operator [SEL_16] - outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] - Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_104] - predicate:((((((v3 = 'ssv3') and v2 is not null) and k1 is not null) and v1 is not null) and k2 is not null) and k3 is not null) (type: boolean) - Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_14] - alias:ss - Statistics:Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions:_col2 (type: string) + Merge Join Operator [MERGEJOIN_105] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"1":"_col0 (type: string)","0":"_col0 (type: string)"} + | outputColumnNames:["_col1","_col2","_col3"] + | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + |<-Map 1 [SIMPLE_EDGE] + | Reduce Output Operator [RS_51] + | key expressions:_col0 (type: string) + | Map-reduce partition columns:_col0 (type: string) + | sort order:+ + | Statistics:Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col1 (type: string), _col2 (type: string), _col3 (type: string) + | Select Operator [SEL_1] + | outputColumnNames:["_col0","_col1","_col2","_col3"] + | Statistics:Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_97] + | predicate:((k1 is not null and v2 is not null) and v3 is not null) (type: boolean) + | Statistics:Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_0] + | alias:cs + | Statistics:Num rows: 170 Data size: 5890 Basic stats: COMPLETE Column stats: NONE + |<-Map 6 [SIMPLE_EDGE] + Reduce Output Operator [RS_53] + key expressions:_col0 (type: string) + Map-reduce partition columns:_col0 (type: string) + sort order:+ + Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_4] + outputColumnNames:["_col0"] + Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_98] + predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_2] + alias:d1 + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: explain SELECT x.key, z.value, y.value @@ -596,256 +596,256 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 13 <- Union 14 (CONTAINS) -Map 17 <- Union 14 (CONTAINS) -Map 5 <- Union 6 (CONTAINS) -Map 9 <- Union 6 (CONTAINS) -Reducer 12 <- Map 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 15 <- Union 14 (SIMPLE_EDGE) -Reducer 16 <- Map 18 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE) +Map 1 <- Union 2 (CONTAINS) +Map 11 <- Union 12 (CONTAINS) +Map 16 <- Union 12 (CONTAINS) +Map 8 <- Union 2 (CONTAINS) +Reducer 13 <- Union 12 (SIMPLE_EDGE) +Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) +Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 3 <- Union 2 (SIMPLE_EDGE) +Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) Reducer 7 <- Union 6 (SIMPLE_EDGE) -Reducer 8 <- Map 10 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 - File Output Operator [FS_61] + Reducer 7 + File Output Operator [FS_59] compressed:false Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Group By Operator [GBY_59] + Group By Operator [GBY_57] | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 3 [SIMPLE_EDGE] - |<-Reducer 2 [CONTAINS] - | Reduce Output Operator [RS_58] + |<-Union 6 [SIMPLE_EDGE] + |<-Reducer 5 [CONTAINS] + | Reduce Output Operator [RS_56] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_57] + | Group By Operator [GBY_55] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_26] + | Select Operator [SEL_25] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | Merge Join Operator [MERGEJOIN_85] + | Merge Join Operator [MERGEJOIN_83] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"1":"_col2 (type: string)","0":"_col0 (type: string)"} - | | outputColumnNames:["_col2","_col3"] + | | keys:{"1":"_col0 (type: string)","0":"_col2 (type: string)"} + | | outputColumnNames:["_col1","_col2"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Map 1 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_22] + | |<-Map 10 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_23] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_1] + | | Select Operator [SEL_14] | | outputColumnNames:["_col0"] | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_76] + | | Filter Operator [FIL_77] | | predicate:key is not null (type: boolean) | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_0] + | | TableScan [TS_13] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 8 [SIMPLE_EDGE] - | Reduce Output Operator [RS_24] + | |<-Reducer 4 [SIMPLE_EDGE] + | Reduce Output Operator [RS_21] | key expressions:_col2 (type: string) | Map-reduce partition columns:_col2 (type: string) | sort order:+ | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE | value expressions:_col1 (type: string) - | Merge Join Operator [MERGEJOIN_84] + | Merge Join Operator [MERGEJOIN_82] | | condition map:[{"":"Inner Join 0 to 1"}] | | keys:{"1":"_col1 (type: string)","0":"_col1 (type: string)"} | | outputColumnNames:["_col1","_col2"] | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | |<-Map 10 [SIMPLE_EDGE] + | |<-Map 9 [SIMPLE_EDGE] | | Reduce Output Operator [RS_18] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string) - | | Select Operator [SEL_14] + | | Select Operator [SEL_12] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_79] + | | Filter Operator [FIL_76] | | predicate:(value is not null and key is not null) (type: boolean) | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_13] + | | TableScan [TS_11] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 7 [SIMPLE_EDGE] + | |<-Reducer 3 [SIMPLE_EDGE] | Reduce Output Operator [RS_16] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) | sort order:+ | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_12] + | Select Operator [SEL_10] | outputColumnNames:["_col1"] | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_11] + | Group By Operator [GBY_9] | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 6 [SIMPLE_EDGE] - | |<-Map 5 [CONTAINS] - | | Reduce Output Operator [RS_10] + | |<-Union 2 [SIMPLE_EDGE] + | |<-Map 1 [CONTAINS] + | | Reduce Output Operator [RS_8] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | sort order:++ | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_9] + | | Group By Operator [GBY_7] | | keys:_col0 (type: string), _col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_3] + | | Select Operator [SEL_1] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_77] + | | Filter Operator [FIL_74] | | predicate:value is not null (type: boolean) | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_2] + | | TableScan [TS_0] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Map 9 [CONTAINS] - | Reduce Output Operator [RS_10] + | |<-Map 8 [CONTAINS] + | Reduce Output Operator [RS_8] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_9] + | Group By Operator [GBY_7] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_5] + | Select Operator [SEL_3] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_78] + | Filter Operator [FIL_75] | predicate:value is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_4] + | TableScan [TS_2] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 12 [CONTAINS] - Reduce Output Operator [RS_58] + |<-Reducer 15 [CONTAINS] + Reduce Output Operator [RS_56] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_57] + Group By Operator [GBY_55] keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_53] + Select Operator [SEL_51] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_87] + Merge Join Operator [MERGEJOIN_85] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"1":"_col2 (type: string)","0":"_col0 (type: string)"} - | outputColumnNames:["_col2","_col3"] + | keys:{"1":"_col0 (type: string)","0":"_col2 (type: string)"} + | outputColumnNames:["_col1","_col2"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Map 11 [SIMPLE_EDGE] + |<-Map 18 [SIMPLE_EDGE] | Reduce Output Operator [RS_49] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_28] + | Select Operator [SEL_40] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_80] + | Filter Operator [FIL_81] | predicate:key is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_27] + | TableScan [TS_39] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 16 [SIMPLE_EDGE] - Reduce Output Operator [RS_51] + |<-Reducer 14 [SIMPLE_EDGE] + Reduce Output Operator [RS_47] key expressions:_col2 (type: string) Map-reduce partition columns:_col2 (type: string) sort order:+ Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE value expressions:_col1 (type: string) - Merge Join Operator [MERGEJOIN_86] + Merge Join Operator [MERGEJOIN_84] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{"1":"_col1 (type: string)","0":"_col1 (type: string)"} | outputColumnNames:["_col1","_col2"] | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - |<-Map 18 [SIMPLE_EDGE] - | Reduce Output Operator [RS_45] + |<-Map 17 [SIMPLE_EDGE] + | Reduce Output Operator [RS_44] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) | sort order:+ | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | value expressions:_col0 (type: string) - | Select Operator [SEL_41] + | Select Operator [SEL_38] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_83] + | Filter Operator [FIL_80] | predicate:(value is not null and key is not null) (type: boolean) | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_40] + | TableScan [TS_37] | alias:x | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 15 [SIMPLE_EDGE] - Reduce Output Operator [RS_43] + |<-Reducer 13 [SIMPLE_EDGE] + Reduce Output Operator [RS_42] key expressions:_col1 (type: string) Map-reduce partition columns:_col1 (type: string) sort order:+ Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_39] + Select Operator [SEL_36] outputColumnNames:["_col1"] Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_38] + Group By Operator [GBY_35] | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - |<-Union 14 [SIMPLE_EDGE] - |<-Map 13 [CONTAINS] - | Reduce Output Operator [RS_37] + |<-Union 12 [SIMPLE_EDGE] + |<-Map 11 [CONTAINS] + | Reduce Output Operator [RS_34] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_36] + | Group By Operator [GBY_33] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_30] + | Select Operator [SEL_27] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_81] + | Filter Operator [FIL_78] | predicate:value is not null (type: boolean) | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_29] + | TableScan [TS_26] | alias:x | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map 17 [CONTAINS] - Reduce Output Operator [RS_37] + |<-Map 16 [CONTAINS] + Reduce Output Operator [RS_34] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_36] + Group By Operator [GBY_33] keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_32] + Select Operator [SEL_29] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_82] + Filter Operator [FIL_79] predicate:value is not null (type: boolean) Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_31] + TableScan [TS_28] alias:y Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -878,500 +878,500 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 11 <- Union 8 (CONTAINS) -Map 15 <- Union 16 (CONTAINS) +Map 1 <- Union 2 (CONTAINS) +Map 10 <- Union 2 (CONTAINS) +Map 13 <- Union 14 (CONTAINS) +Map 20 <- Union 14 (CONTAINS) Map 21 <- Union 16 (CONTAINS) -Map 22 <- Union 18 (CONTAINS) -Map 26 <- Union 27 (CONTAINS) +Map 24 <- Union 25 (CONTAINS) +Map 33 <- Union 25 (CONTAINS) Map 34 <- Union 27 (CONTAINS) Map 35 <- Union 29 (CONTAINS) -Map 36 <- Union 31 (CONTAINS) -Map 7 <- Union 8 (CONTAINS) -Reducer 10 <- Map 12 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 17 <- Union 16 (SIMPLE_EDGE), Union 18 (CONTAINS) -Reducer 19 <- Union 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 20 <- Map 23 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 25 <- Map 24 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE), Union 5 (CONTAINS) +Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS) +Reducer 17 <- Union 16 (SIMPLE_EDGE) +Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) +Reducer 19 <- Map 23 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 26 <- Union 25 (SIMPLE_EDGE), Union 27 (CONTAINS) Reducer 28 <- Union 27 (SIMPLE_EDGE), Union 29 (CONTAINS) -Reducer 30 <- Union 29 (SIMPLE_EDGE), Union 31 (CONTAINS) -Reducer 32 <- Union 31 (SIMPLE_EDGE) -Reducer 33 <- Map 37 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) -Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) +Reducer 3 <- Union 2 (SIMPLE_EDGE) +Reducer 30 <- Union 29 (SIMPLE_EDGE) +Reducer 31 <- Map 36 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) +Reducer 32 <- Map 37 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE), Union 8 (CONTAINS) +Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) Reducer 9 <- Union 8 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_122] + Reducer 9 + File Output Operator [FS_119] compressed:false Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Group By Operator [GBY_120] + Group By Operator [GBY_117] | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 5 [SIMPLE_EDGE] - |<-Reducer 25 [CONTAINS] - | Reduce Output Operator [RS_119] + |<-Union 8 [SIMPLE_EDGE] + |<-Reducer 32 [CONTAINS] + | Reduce Output Operator [RS_116] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_118] + | Group By Operator [GBY_115] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_114] + | Select Operator [SEL_111] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | Merge Join Operator [MERGEJOIN_170] + | Merge Join Operator [MERGEJOIN_167] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"1":"_col2 (type: string)","0":"_col0 (type: string)"} - | | outputColumnNames:["_col1","_col4"] + | | keys:{"1":"_col0 (type: string)","0":"_col2 (type: string)"} + | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Map 24 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_110] + | |<-Map 37 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_109] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col1 (type: string) - | | Select Operator [SEL_71] + | | Select Operator [SEL_100] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_159] + | | Filter Operator [FIL_161] | | predicate:key is not null (type: boolean) | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_70] + | | TableScan [TS_99] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 33 [SIMPLE_EDGE] - | Reduce Output Operator [RS_112] + | |<-Reducer 31 [SIMPLE_EDGE] + | Reduce Output Operator [RS_107] | key expressions:_col2 (type: string) | Map-reduce partition columns:_col2 (type: string) | sort order:+ | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - | Merge Join Operator [MERGEJOIN_169] + | Merge Join Operator [MERGEJOIN_166] | | condition map:[{"":"Inner Join 0 to 1"}] | | keys:{"1":"_col1 (type: string)","0":"_col1 (type: string)"} | | outputColumnNames:["_col2"] | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - | |<-Map 37 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_106] + | |<-Map 36 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_104] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string) - | | Select Operator [SEL_102] + | | Select Operator [SEL_98] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_164] + | | Filter Operator [FIL_160] | | predicate:(value is not null and key is not null) (type: boolean) | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_101] + | | TableScan [TS_97] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 32 [SIMPLE_EDGE] - | Reduce Output Operator [RS_104] + | |<-Reducer 30 [SIMPLE_EDGE] + | Reduce Output Operator [RS_102] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) | sort order:+ | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_100] + | Select Operator [SEL_96] | outputColumnNames:["_col1"] | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_99] + | Group By Operator [GBY_95] | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - | |<-Union 31 [SIMPLE_EDGE] - | |<-Reducer 30 [CONTAINS] - | | Reduce Output Operator [RS_98] + | |<-Union 29 [SIMPLE_EDGE] + | |<-Map 35 [CONTAINS] + | | Reduce Output Operator [RS_94] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | sort order:++ | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_97] + | | Group By Operator [GBY_93] | | keys:_col0 (type: string), _col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_90] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 29 [SIMPLE_EDGE] - | | |<-Map 35 [CONTAINS] - | | | Reduce Output Operator [RS_89] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_88] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_84] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_162] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_83] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Reducer 28 [CONTAINS] - | | Reduce Output Operator [RS_89] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_88] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_81] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 27 [SIMPLE_EDGE] - | | |<-Map 34 [CONTAINS] - | | | Reduce Output Operator [RS_80] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_79] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_75] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_161] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_74] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 26 [CONTAINS] - | | Reduce Output Operator [RS_80] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_79] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_73] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_160] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_72] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Map 36 [CONTAINS] - | Reduce Output Operator [RS_98] + | | Select Operator [SEL_89] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_159] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_88] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 28 [CONTAINS] + | Reduce Output Operator [RS_94] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_97] + | Group By Operator [GBY_93] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_93] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_163] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_92] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_119] + | Group By Operator [GBY_86] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + | |<-Union 27 [SIMPLE_EDGE] + | |<-Map 34 [CONTAINS] + | | Reduce Output Operator [RS_85] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_84] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_80] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_158] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_79] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 26 [CONTAINS] + | Reduce Output Operator [RS_85] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_84] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_77] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | |<-Union 25 [SIMPLE_EDGE] + | |<-Map 24 [CONTAINS] + | | Reduce Output Operator [RS_76] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_75] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_69] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_156] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_68] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Map 33 [CONTAINS] + | Reduce Output Operator [RS_76] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_75] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_71] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_157] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_70] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 7 [CONTAINS] + Reduce Output Operator [RS_116] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_118] + Group By Operator [GBY_115] keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_68] + Group By Operator [GBY_66] | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 3 [SIMPLE_EDGE] - |<-Reducer 14 [CONTAINS] - | Reduce Output Operator [RS_67] + |<-Union 6 [SIMPLE_EDGE] + |<-Reducer 5 [CONTAINS] + | Reduce Output Operator [RS_65] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_66] + | Group By Operator [GBY_64] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_62] + | Select Operator [SEL_25] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | Merge Join Operator [MERGEJOIN_168] + | Merge Join Operator [MERGEJOIN_163] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"1":"_col2 (type: string)","0":"_col0 (type: string)"} - | | outputColumnNames:["_col1","_col4"] + | | keys:{"1":"_col0 (type: string)","0":"_col2 (type: string)"} + | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Map 13 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_58] + | |<-Map 12 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_23] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col1 (type: string) - | | Select Operator [SEL_28] + | | Select Operator [SEL_14] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_154] + | | Filter Operator [FIL_150] | | predicate:key is not null (type: boolean) | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_27] + | | TableScan [TS_13] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 20 [SIMPLE_EDGE] - | Reduce Output Operator [RS_60] + | |<-Reducer 4 [SIMPLE_EDGE] + | Reduce Output Operator [RS_21] | key expressions:_col2 (type: string) | Map-reduce partition columns:_col2 (type: string) | sort order:+ - | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | Merge Join Operator [MERGEJOIN_167] + | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE + | Merge Join Operator [MERGEJOIN_162] | | condition map:[{"":"Inner Join 0 to 1"}] | | keys:{"1":"_col1 (type: string)","0":"_col1 (type: string)"} | | outputColumnNames:["_col2"] - | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | |<-Map 23 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_54] + | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE + | |<-Map 11 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_18] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string) - | | Select Operator [SEL_50] + | | Select Operator [SEL_12] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_158] + | | Filter Operator [FIL_149] | | predicate:(value is not null and key is not null) (type: boolean) | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_49] + | | TableScan [TS_11] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 19 [SIMPLE_EDGE] - | Reduce Output Operator [RS_52] + | |<-Reducer 3 [SIMPLE_EDGE] + | Reduce Output Operator [RS_16] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) | sort order:+ - | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_48] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_10] | outputColumnNames:["_col1"] - | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_47] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_9] | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | |<-Union 18 [SIMPLE_EDGE] - | |<-Map 22 [CONTAINS] - | | Reduce Output Operator [RS_46] + | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | |<-Union 2 [SIMPLE_EDGE] + | |<-Map 1 [CONTAINS] + | | Reduce Output Operator [RS_8] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | sort order:++ - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_45] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_7] | | keys:_col0 (type: string), _col1 (type: string) | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_41] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_1] | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_157] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_147] | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_40] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 17 [CONTAINS] - | Reduce Output Operator [RS_46] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_0] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Map 10 [CONTAINS] + | Reduce Output Operator [RS_8] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ - | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_45] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_7] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_38] - | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 16 [SIMPLE_EDGE] - | |<-Map 21 [CONTAINS] - | | Reduce Output Operator [RS_37] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_36] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_32] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_156] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_31] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Map 15 [CONTAINS] - | Reduce Output Operator [RS_37] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_36] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_30] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_155] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_29] - | alias:x - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 2 [CONTAINS] - Reduce Output Operator [RS_67] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_3] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_148] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_2] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 19 [CONTAINS] + Reduce Output Operator [RS_65] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_66] + Group By Operator [GBY_64] keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_26] + Select Operator [SEL_60] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_166] + Merge Join Operator [MERGEJOIN_165] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"1":"_col2 (type: string)","0":"_col0 (type: string)"} - | outputColumnNames:["_col1","_col4"] + | keys:{"1":"_col0 (type: string)","0":"_col2 (type: string)"} + | outputColumnNames:["_col2","_col5"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_22] + |<-Map 23 [SIMPLE_EDGE] + | Reduce Output Operator [RS_58] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE | value expressions:_col1 (type: string) - | Select Operator [SEL_1] + | Select Operator [SEL_49] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_150] + | Filter Operator [FIL_155] | predicate:key is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_0] + | TableScan [TS_48] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 10 [SIMPLE_EDGE] - Reduce Output Operator [RS_24] + |<-Reducer 18 [SIMPLE_EDGE] + Reduce Output Operator [RS_56] key expressions:_col2 (type: string) Map-reduce partition columns:_col2 (type: string) sort order:+ - Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_165] + Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE + Merge Join Operator [MERGEJOIN_164] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{"1":"_col1 (type: string)","0":"_col1 (type: string)"} | outputColumnNames:["_col2"] - | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - |<-Map 12 [SIMPLE_EDGE] - | Reduce Output Operator [RS_18] + | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE + |<-Map 22 [SIMPLE_EDGE] + | Reduce Output Operator [RS_53] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) | sort order:+ | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | value expressions:_col0 (type: string) - | Select Operator [SEL_14] + | Select Operator [SEL_47] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_153] + | Filter Operator [FIL_154] | predicate:(value is not null and key is not null) (type: boolean) | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_13] + | TableScan [TS_46] | alias:x | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 9 [SIMPLE_EDGE] - Reduce Output Operator [RS_16] + |<-Reducer 17 [SIMPLE_EDGE] + Reduce Output Operator [RS_51] key expressions:_col1 (type: string) Map-reduce partition columns:_col1 (type: string) sort order:+ - Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_12] + Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_45] outputColumnNames:["_col1"] - Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_11] + Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_44] | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - |<-Union 8 [SIMPLE_EDGE] - |<-Map 11 [CONTAINS] - | Reduce Output Operator [RS_10] + | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + |<-Union 16 [SIMPLE_EDGE] + |<-Map 21 [CONTAINS] + | Reduce Output Operator [RS_43] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_9] + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_42] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_5] + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_38] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_152] + | Filter Operator [FIL_153] | predicate:value is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_4] + | TableScan [TS_37] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 7 [CONTAINS] - Reduce Output Operator [RS_10] + |<-Reducer 15 [CONTAINS] + Reduce Output Operator [RS_43] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ - Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_9] + Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_42] keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_3] - outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_151] - predicate:value is not null (type: boolean) - Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_2] - alias:x - Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_35] + | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + |<-Union 14 [SIMPLE_EDGE] + |<-Map 13 [CONTAINS] + | Reduce Output Operator [RS_34] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_33] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_27] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_151] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_26] + | alias:x + | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + |<-Map 20 [CONTAINS] + Reduce Output Operator [RS_34] + key expressions:_col0 (type: string), _col1 (type: string) + Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + sort order:++ + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_33] + keys:_col0 (type: string), _col1 (type: string) + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_29] + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_152] + predicate:value is not null (type: boolean) + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_28] + alias:y + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: EXPLAIN SELECT x.key, z.value, y.value @@ -1386,69 +1386,69 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 2 <- Map 1 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Map 2 - File Output Operator [FS_19] + Map 1 + File Output Operator [FS_18] compressed:false Statistics:Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Select Operator [SEL_18] + Select Operator [SEL_17] outputColumnNames:["_col0","_col1","_col2"] Statistics:Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE - Map Join Operator [MAPJOIN_29] + Map Join Operator [MAPJOIN_28] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 1":"_col0 (type: string)","Map 2":"_col3 (type: string)"} - | outputColumnNames:["_col1","_col2","_col5"] + | keys:{"Map 1":"_col3 (type: string)","Map 3":"_col0 (type: string)"} + | outputColumnNames:["_col0","_col3","_col6"] | Statistics:Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE - |<-Map 1 [BROADCAST_EDGE] - | Reduce Output Operator [RS_14] + |<-Map 3 [BROADCAST_EDGE] + | Reduce Output Operator [RS_15] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE | value expressions:_col1 (type: string) - | Select Operator [SEL_1] + | Select Operator [SEL_6] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_25] + | Filter Operator [FIL_26] | predicate:key is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_0] + | TableScan [TS_5] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map Join Operator [MAPJOIN_28] + |<-Map Join Operator [MAPJOIN_27] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 2":"_col0 (type: string)","Map 3":"_col1 (type: string)"} + | keys:{"Map 1":"_col0 (type: string)","Map 2":"_col1 (type: string)"} | outputColumnNames:["_col0","_col3"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Map 3 [BROADCAST_EDGE] + |<-Map 2 [BROADCAST_EDGE] | Reduce Output Operator [RS_10] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) | sort order:+ | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | value expressions:_col0 (type: string) - | Select Operator [SEL_6] + | Select Operator [SEL_4] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_27] + | Filter Operator [FIL_25] | predicate:(value is not null and key is not null) (type: boolean) | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_5] + | TableScan [TS_3] | alias:x | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_4] + |<-Select Operator [SEL_2] outputColumnNames:["_col0"] Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_26] + Filter Operator [FIL_24] predicate:value is not null (type: boolean) Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_2] + TableScan [TS_0] alias:z Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -1513,70 +1513,70 @@ Plan optimized by CBO. Vertex dependency in root stage Map 10 <- Map 9 (BROADCAST_EDGE) Map 2 <- Map 1 (BROADCAST_EDGE) -Map 5 <- Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE) -Reducer 6 <- Map 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Map 3 <- Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE) +Reducer 4 <- Map 3 (SIMPLE_EDGE) +Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 7 - File Output Operator [FS_71] + Reducer 5 + File Output Operator [FS_69] compressed:false Statistics:Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Limit [LIM_70] + Limit [LIM_68] Number of rows:100 Statistics:Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_69] + Select Operator [SEL_67] | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] | Statistics:Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 6 [SIMPLE_EDGE] - Reduce Output Operator [RS_68] + |<-Reducer 4 [SIMPLE_EDGE] + Reduce Output Operator [RS_66] key expressions:_col0 (type: string), _col1 (type: string), _col2 (type: string) sort order:+++ Statistics:Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE value expressions:_col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint) - Group By Operator [GBY_66] + Group By Operator [GBY_64] | aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"] | keys:KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] | Statistics:Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE - |<-Map 5 [SIMPLE_EDGE] - Reduce Output Operator [RS_65] + |<-Map 3 [SIMPLE_EDGE] + Reduce Output Operator [RS_63] key expressions:_col0 (type: string), _col1 (type: string), _col2 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string), _col2 (type: string) sort order:+++ Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE value expressions:_col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint) - Group By Operator [GBY_64] + Group By Operator [GBY_62] aggregations:["count(_col3)","count(_col4)","count(_col5)"] keys:_col0 (type: string), _col1 (type: string), _col2 (type: string) outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_62] + Select Operator [SEL_60] outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE - Map Join Operator [MAPJOIN_113] + Map Join Operator [MAPJOIN_111] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 2":"_col1 (type: string), _col3 (type: string)","Map 5":"_col15 (type: string), _col17 (type: string)"} + | keys:{"Map 2":"_col1 (type: string), _col3 (type: string)","Map 3":"_col15 (type: string), _col17 (type: string)"} | outputColumnNames:["_col2","_col3","_col12","_col13","_col20","_col21"] | Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE |<-Map 2 [BROADCAST_EDGE] - | Reduce Output Operator [RS_58] + | Reduce Output Operator [RS_56] | key expressions:_col1 (type: string), _col3 (type: string) | Map-reduce partition columns:_col1 (type: string), _col3 (type: string) | sort order:++ | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE | value expressions:_col2 (type: string) - | Map Join Operator [MAPJOIN_107] + | Map Join Operator [MAPJOIN_105] | | condition map:[{"":"Inner Join 0 to 1"}] | | keys:{"Map 1":"_col0 (type: string)","Map 2":"_col0 (type: string)"} | | outputColumnNames:["_col1","_col2","_col3"] | | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE | |<-Map 1 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_53] + | | Reduce Output Operator [RS_51] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ @@ -1585,7 +1585,7 @@ Stage-0 | | Select Operator [SEL_1] | | outputColumnNames:["_col0","_col1","_col2","_col3"] | | Statistics:Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_99] + | | Filter Operator [FIL_97] | | predicate:((k1 is not null and v2 is not null) and v3 is not null) (type: boolean) | | Statistics:Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE | | TableScan [TS_0] @@ -1594,125 +1594,125 @@ Stage-0 | |<-Select Operator [SEL_4] | outputColumnNames:["_col0"] | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_100] + | Filter Operator [FIL_98] | predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE | TableScan [TS_2] | alias:d1 | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_51] + |<-Select Operator [SEL_49] outputColumnNames:["_col14","_col15","_col17","_col6","_col7"] Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE - Map Join Operator [MAPJOIN_112] + Map Join Operator [MAPJOIN_110] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 10":"_col2 (type: string), _col4 (type: string)","Map 5":"_col8 (type: string), _col10 (type: string)"} - | outputColumnNames:["_col6","_col7","_col14","_col15","_col17"] + | keys:{"Map 10":"_col2 (type: string), _col4 (type: string)","Map 3":"_col4 (type: string), _col6 (type: string)"} + | outputColumnNames:["_col2","_col3","_col14","_col15","_col17"] | Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE |<-Map 10 [BROADCAST_EDGE] - | Reduce Output Operator [RS_49] + | Reduce Output Operator [RS_47] | key expressions:_col2 (type: string), _col4 (type: string) | Map-reduce partition columns:_col2 (type: string), _col4 (type: string) | sort order:++ | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE | value expressions:_col3 (type: string), _col5 (type: string) - | Map Join Operator [MAPJOIN_111] + | Map Join Operator [MAPJOIN_109] | | condition map:[{"":"Inner Join 0 to 1"}] | | keys:{"Map 10":"_col0 (type: string)","Map 9":"_col0 (type: string)"} | | outputColumnNames:["_col2","_col3","_col4","_col5"] | | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE | |<-Map 9 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_36] + | | Reduce Output Operator [RS_24] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) - | | Select Operator [SEL_31] + | | Select Operator [SEL_19] | | outputColumnNames:["_col0","_col2","_col3","_col4","_col5"] | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_105] + | | Filter Operator [FIL_103] | | predicate:((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_29] + | | TableScan [TS_17] | | alias:sr | | Statistics:Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_34] + | |<-Select Operator [SEL_22] | outputColumnNames:["_col0"] | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_106] + | Filter Operator [FIL_104] | predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_32] + | TableScan [TS_20] | alias:d1 | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map Join Operator [MAPJOIN_110] + |<-Map Join Operator [MAPJOIN_108] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 3":"_col1 (type: string)","Map 5":"_col5 (type: string)"} - | outputColumnNames:["_col6","_col7","_col8","_col10"] + | keys:{"Map 3":"_col3 (type: string)","Map 8":"_col1 (type: string)"} + | outputColumnNames:["_col2","_col3","_col4","_col6"] | Statistics:Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE - |<-Map 3 [BROADCAST_EDGE] + |<-Map 8 [BROADCAST_EDGE] | Reduce Output Operator [RS_42] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) | sort order:+ | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_7] + | Select Operator [SEL_16] | outputColumnNames:["_col1"] | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_101] + | Filter Operator [FIL_102] | predicate:((key = 'src1key') and value is not null) (type: boolean) | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_5] + | TableScan [TS_14] | alias:src1 | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map Join Operator [MAPJOIN_109] + |<-Map Join Operator [MAPJOIN_107] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 5":"_col2 (type: string)","Map 4":"_col0 (type: string)"} - | outputColumnNames:["_col4","_col5","_col6","_col8"] + | keys:{"Map 3":"_col2 (type: string)","Map 7":"_col0 (type: string)"} + | outputColumnNames:["_col2","_col3","_col4","_col6"] | Statistics:Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - |<-Map 4 [BROADCAST_EDGE] - | Reduce Output Operator [RS_24] + |<-Map 7 [BROADCAST_EDGE] + | Reduce Output Operator [RS_37] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_10] + | Select Operator [SEL_13] | outputColumnNames:["_col0"] | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_102] + | Filter Operator [FIL_101] | predicate:((value = 'd1value') and key is not null) (type: boolean) | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_8] + | TableScan [TS_11] | alias:d1 | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map Join Operator [MAPJOIN_108] + |<-Map Join Operator [MAPJOIN_106] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 5":"_col1 (type: string)","Map 8":"_col3 (type: string)"} + | keys:{"Map 3":"_col1 (type: string)","Map 6":"_col3 (type: string)"} | outputColumnNames:["_col2","_col3","_col4","_col6"] | Statistics:Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - |<-Map 8 [BROADCAST_EDGE] - | Reduce Output Operator [RS_20] + |<-Map 6 [BROADCAST_EDGE] + | Reduce Output Operator [RS_32] | key expressions:_col3 (type: string) | Map-reduce partition columns:_col3 (type: string) | sort order:+ | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE | value expressions:_col0 (type: string), _col1 (type: string), _col2 (type: string), _col4 (type: string) - | Select Operator [SEL_16] + | Select Operator [SEL_10] | outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_104] + | Filter Operator [FIL_100] | predicate:((((((v3 = 'ssv3') and v2 is not null) and k1 is not null) and v1 is not null) and k2 is not null) and k3 is not null) (type: boolean) | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_14] + | TableScan [TS_8] | alias:ss | Statistics:Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_13] + |<-Select Operator [SEL_7] outputColumnNames:["_col1"] Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_103] + Filter Operator [FIL_99] predicate:((key = 'srcpartkey') and value is not null) (type: boolean) Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_11] + TableScan [TS_5] alias:srcpart Statistics:Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE @@ -1737,50 +1737,50 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Reducer 6 (BROADCAST_EDGE), Union 2 (CONTAINS) -Map 10 <- Union 11 (CONTAINS) -Map 13 <- Union 11 (CONTAINS) -Map 4 <- Union 5 (CONTAINS) -Map 7 <- Union 5 (CONTAINS) -Map 9 <- Reducer 12 (BROADCAST_EDGE), Union 2 (CONTAINS) -Reducer 12 <- Map 14 (BROADCAST_EDGE), Union 11 (SIMPLE_EDGE) -Reducer 3 <- Union 2 (SIMPLE_EDGE) -Reducer 6 <- Map 8 (BROADCAST_EDGE), Union 5 (SIMPLE_EDGE) +Map 1 <- Union 2 (CONTAINS) +Map 12 <- Union 10 (CONTAINS) +Map 14 <- Reducer 11 (BROADCAST_EDGE), Union 7 (CONTAINS) +Map 4 <- Union 2 (CONTAINS) +Map 6 <- Reducer 3 (BROADCAST_EDGE), Union 7 (CONTAINS) +Map 9 <- Union 10 (CONTAINS) +Reducer 11 <- Map 13 (BROADCAST_EDGE), Union 10 (SIMPLE_EDGE) +Reducer 3 <- Map 5 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) +Reducer 8 <- Union 7 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 - File Output Operator [FS_61] + Reducer 8 + File Output Operator [FS_59] compressed:false Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Group By Operator [GBY_59] + Group By Operator [GBY_57] | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 2 [SIMPLE_EDGE] - |<-Map 1 [CONTAINS] - | Reduce Output Operator [RS_58] + |<-Union 7 [SIMPLE_EDGE] + |<-Map 14 [CONTAINS] + | Reduce Output Operator [RS_56] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_57] + | Group By Operator [GBY_55] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_26] + | Select Operator [SEL_51] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_85] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 1":"_col0 (type: string)","Reducer 6":"_col2 (type: string)"} - | | outputColumnNames:["_col2","_col3"] + | | keys:{"Reducer 11":"_col2 (type: string)","Map 14":"_col0 (type: string)"} + | | outputColumnNames:["_col1","_col2"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 6 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_24] + | |<-Reducer 11 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_47] | | key expressions:_col2 (type: string) | | Map-reduce partition columns:_col2 (type: string) | | sort order:+ @@ -1788,179 +1788,179 @@ Stage-0 | | value expressions:_col1 (type: string) | | Map Join Operator [MAPJOIN_84] | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Reducer 6":"_col1 (type: string)","Map 8":"_col1 (type: string)"} + | | | keys:{"Reducer 11":"_col1 (type: string)","Map 13":"_col1 (type: string)"} | | | outputColumnNames:["_col1","_col2"] | | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 8 [BROADCAST_EDGE] - | | | Reduce Output Operator [RS_18] + | | |<-Map 13 [BROADCAST_EDGE] + | | | Reduce Output Operator [RS_44] | | | key expressions:_col1 (type: string) | | | Map-reduce partition columns:_col1 (type: string) | | | sort order:+ | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | | | value expressions:_col0 (type: string) - | | | Select Operator [SEL_14] + | | | Select Operator [SEL_38] | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_79] + | | | Filter Operator [FIL_80] | | | predicate:(value is not null and key is not null) (type: boolean) | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_13] + | | | TableScan [TS_37] | | | alias:x | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Select Operator [SEL_12] + | | |<-Select Operator [SEL_36] | | outputColumnNames:["_col1"] | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_11] + | | Group By Operator [GBY_35] | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 5 [SIMPLE_EDGE] - | | |<-Map 4 [CONTAINS] - | | | Reduce Output Operator [RS_10] + | | |<-Union 10 [SIMPLE_EDGE] + | | |<-Map 12 [CONTAINS] + | | | Reduce Output Operator [RS_34] | | | key expressions:_col0 (type: string), _col1 (type: string) | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | | sort order:++ | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_9] + | | | Group By Operator [GBY_33] | | | keys:_col0 (type: string), _col1 (type: string) | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_3] + | | | Select Operator [SEL_29] | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_77] + | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | | Filter Operator [FIL_79] | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_2] - | | | alias:x - | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 7 [CONTAINS] - | | Reduce Output Operator [RS_10] + | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | | TableScan [TS_28] + | | | alias:y + | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | | |<-Map 9 [CONTAINS] + | | Reduce Output Operator [RS_34] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | sort order:++ | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_9] + | | Group By Operator [GBY_33] | | keys:_col0 (type: string), _col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_5] + | | Select Operator [SEL_27] | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE | | Filter Operator [FIL_78] | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_4] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_1] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_26] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Select Operator [SEL_40] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_76] + | Filter Operator [FIL_81] | predicate:key is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_0] + | TableScan [TS_39] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 9 [CONTAINS] - Reduce Output Operator [RS_58] + |<-Map 6 [CONTAINS] + Reduce Output Operator [RS_56] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_57] + Group By Operator [GBY_55] keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_53] + Select Operator [SEL_25] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Map Join Operator [MAPJOIN_87] + Map Join Operator [MAPJOIN_83] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Reducer 12":"_col2 (type: string)","Map 9":"_col0 (type: string)"} - | outputColumnNames:["_col2","_col3"] + | keys:{"Reducer 3":"_col2 (type: string)","Map 6":"_col0 (type: string)"} + | outputColumnNames:["_col1","_col2"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 12 [BROADCAST_EDGE] - | Reduce Output Operator [RS_51] + |<-Reducer 3 [BROADCAST_EDGE] + | Reduce Output Operator [RS_21] | key expressions:_col2 (type: string) | Map-reduce partition columns:_col2 (type: string) | sort order:+ | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE | value expressions:_col1 (type: string) - | Map Join Operator [MAPJOIN_86] + | Map Join Operator [MAPJOIN_82] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 14":"_col1 (type: string)","Reducer 12":"_col1 (type: string)"} + | | keys:{"Reducer 3":"_col1 (type: string)","Map 5":"_col1 (type: string)"} | | outputColumnNames:["_col1","_col2"] | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | |<-Map 14 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_45] + | |<-Map 5 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_18] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string) - | | Select Operator [SEL_41] + | | Select Operator [SEL_12] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_83] + | | Filter Operator [FIL_76] | | predicate:(value is not null and key is not null) (type: boolean) | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_40] + | | TableScan [TS_11] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_39] + | |<-Select Operator [SEL_10] | outputColumnNames:["_col1"] | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_38] + | Group By Operator [GBY_9] | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 11 [SIMPLE_EDGE] - | |<-Map 13 [CONTAINS] - | | Reduce Output Operator [RS_37] + | |<-Union 2 [SIMPLE_EDGE] + | |<-Map 1 [CONTAINS] + | | Reduce Output Operator [RS_8] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | sort order:++ | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_36] + | | Group By Operator [GBY_7] | | keys:_col0 (type: string), _col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_32] + | | Select Operator [SEL_1] | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_82] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_74] | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_31] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Map 10 [CONTAINS] - | Reduce Output Operator [RS_37] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_0] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Map 4 [CONTAINS] + | Reduce Output Operator [RS_8] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_36] + | Group By Operator [GBY_7] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_30] + | Select Operator [SEL_3] | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_81] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_75] | predicate:value is not null (type: boolean) - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_29] - | alias:x - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_28] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_2] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Select Operator [SEL_14] outputColumnNames:["_col0"] Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_80] + Filter Operator [FIL_77] predicate:key is not null (type: boolean) Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_27] + TableScan [TS_13] alias:y Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -1993,456 +1993,456 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 1 <- Reducer 8 (BROADCAST_EDGE), Union 2 (CONTAINS) -Map 11 <- Reducer 16 (BROADCAST_EDGE), Union 2 (CONTAINS) -Map 12 <- Union 13 (CONTAINS) -Map 17 <- Union 13 (CONTAINS) -Map 18 <- Union 15 (CONTAINS) -Map 20 <- Reducer 27 (BROADCAST_EDGE), Union 4 (CONTAINS) -Map 21 <- Union 22 (CONTAINS) -Map 28 <- Union 22 (CONTAINS) -Map 29 <- Union 24 (CONTAINS) -Map 30 <- Union 26 (CONTAINS) -Map 6 <- Union 7 (CONTAINS) -Map 9 <- Union 7 (CONTAINS) -Reducer 14 <- Union 13 (SIMPLE_EDGE), Union 15 (CONTAINS) -Reducer 16 <- Map 19 (BROADCAST_EDGE), Union 15 (SIMPLE_EDGE) -Reducer 23 <- Union 22 (SIMPLE_EDGE), Union 24 (CONTAINS) -Reducer 25 <- Union 24 (SIMPLE_EDGE), Union 26 (CONTAINS) -Reducer 27 <- Map 31 (BROADCAST_EDGE), Union 26 (SIMPLE_EDGE) -Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) -Reducer 5 <- Union 4 (SIMPLE_EDGE) -Reducer 8 <- Map 10 (BROADCAST_EDGE), Union 7 (SIMPLE_EDGE) +Map 1 <- Union 2 (CONTAINS) +Map 11 <- Union 12 (CONTAINS) +Map 16 <- Union 12 (CONTAINS) +Map 17 <- Union 14 (CONTAINS) +Map 19 <- Reducer 15 (BROADCAST_EDGE), Union 7 (CONTAINS) +Map 20 <- Union 21 (CONTAINS) +Map 27 <- Union 21 (CONTAINS) +Map 28 <- Union 23 (CONTAINS) +Map 29 <- Union 25 (CONTAINS) +Map 31 <- Reducer 26 (BROADCAST_EDGE), Union 9 (CONTAINS) +Map 4 <- Union 2 (CONTAINS) +Map 6 <- Reducer 3 (BROADCAST_EDGE), Union 7 (CONTAINS) +Reducer 10 <- Union 9 (SIMPLE_EDGE) +Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS) +Reducer 15 <- Map 18 (BROADCAST_EDGE), Union 14 (SIMPLE_EDGE) +Reducer 22 <- Union 21 (SIMPLE_EDGE), Union 23 (CONTAINS) +Reducer 24 <- Union 23 (SIMPLE_EDGE), Union 25 (CONTAINS) +Reducer 26 <- Map 30 (BROADCAST_EDGE), Union 25 (SIMPLE_EDGE) +Reducer 3 <- Map 5 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) +Reducer 8 <- Union 7 (SIMPLE_EDGE), Union 9 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 - File Output Operator [FS_122] + Reducer 10 + File Output Operator [FS_119] compressed:false Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Group By Operator [GBY_120] + Group By Operator [GBY_117] | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 4 [SIMPLE_EDGE] - |<-Map 20 [CONTAINS] - | Reduce Output Operator [RS_119] + |<-Union 9 [SIMPLE_EDGE] + |<-Map 31 [CONTAINS] + | Reduce Output Operator [RS_116] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_118] + | Group By Operator [GBY_115] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_114] + | Select Operator [SEL_111] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_170] + | Map Join Operator [MAPJOIN_167] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 20":"_col0 (type: string)","Reducer 27":"_col2 (type: string)"} - | | outputColumnNames:["_col1","_col4"] + | | keys:{"Map 31":"_col0 (type: string)","Reducer 26":"_col2 (type: string)"} + | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 27 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_112] + | |<-Reducer 26 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_107] | | key expressions:_col2 (type: string) | | Map-reduce partition columns:_col2 (type: string) | | sort order:+ | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - | | Map Join Operator [MAPJOIN_169] + | | Map Join Operator [MAPJOIN_166] | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Map 31":"_col1 (type: string)","Reducer 27":"_col1 (type: string)"} + | | | keys:{"Map 30":"_col1 (type: string)","Reducer 26":"_col1 (type: string)"} | | | outputColumnNames:["_col2"] | | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 31 [BROADCAST_EDGE] - | | | Reduce Output Operator [RS_106] + | | |<-Map 30 [BROADCAST_EDGE] + | | | Reduce Output Operator [RS_104] | | | key expressions:_col1 (type: string) | | | Map-reduce partition columns:_col1 (type: string) | | | sort order:+ | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | | | value expressions:_col0 (type: string) - | | | Select Operator [SEL_102] + | | | Select Operator [SEL_98] | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_164] + | | | Filter Operator [FIL_160] | | | predicate:(value is not null and key is not null) (type: boolean) | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_101] + | | | TableScan [TS_97] | | | alias:x | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Select Operator [SEL_100] + | | |<-Select Operator [SEL_96] | | outputColumnNames:["_col1"] | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_99] + | | Group By Operator [GBY_95] | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 26 [SIMPLE_EDGE] - | | |<-Map 30 [CONTAINS] - | | | Reduce Output Operator [RS_98] + | | |<-Union 25 [SIMPLE_EDGE] + | | |<-Reducer 24 [CONTAINS] + | | | Reduce Output Operator [RS_94] | | | key expressions:_col0 (type: string), _col1 (type: string) | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | | sort order:++ | | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_97] + | | | Group By Operator [GBY_93] | | | keys:_col0 (type: string), _col1 (type: string) | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_93] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_163] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_92] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Reducer 25 [CONTAINS] - | | Reduce Output Operator [RS_98] + | | | Group By Operator [GBY_86] + | | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | | | outputColumnNames:["_col0","_col1"] + | | | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + | | | |<-Union 23 [SIMPLE_EDGE] + | | | |<-Reducer 22 [CONTAINS] + | | | | Reduce Output Operator [RS_85] + | | | | key expressions:_col0 (type: string), _col1 (type: string) + | | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | | | sort order:++ + | | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | | | Group By Operator [GBY_84] + | | | | keys:_col0 (type: string), _col1 (type: string) + | | | | outputColumnNames:["_col0","_col1"] + | | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | | | Group By Operator [GBY_77] + | | | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | | | | outputColumnNames:["_col0","_col1"] + | | | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | | | | |<-Union 21 [SIMPLE_EDGE] + | | | | |<-Map 20 [CONTAINS] + | | | | | Reduce Output Operator [RS_76] + | | | | | key expressions:_col0 (type: string), _col1 (type: string) + | | | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | | | | sort order:++ + | | | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | | | | Group By Operator [GBY_75] + | | | | | keys:_col0 (type: string), _col1 (type: string) + | | | | | outputColumnNames:["_col0","_col1"] + | | | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | | | | Select Operator [SEL_69] + | | | | | outputColumnNames:["_col0","_col1"] + | | | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | | | | Filter Operator [FIL_156] + | | | | | predicate:value is not null (type: boolean) + | | | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | | | | TableScan [TS_68] + | | | | | alias:x + | | | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | | | | |<-Map 27 [CONTAINS] + | | | | Reduce Output Operator [RS_76] + | | | | key expressions:_col0 (type: string), _col1 (type: string) + | | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | | | sort order:++ + | | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | | | Group By Operator [GBY_75] + | | | | keys:_col0 (type: string), _col1 (type: string) + | | | | outputColumnNames:["_col0","_col1"] + | | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | | | Select Operator [SEL_71] + | | | | outputColumnNames:["_col0","_col1"] + | | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | | | Filter Operator [FIL_157] + | | | | predicate:value is not null (type: boolean) + | | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | | | TableScan [TS_70] + | | | | alias:y + | | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | | | |<-Map 28 [CONTAINS] + | | | Reduce Output Operator [RS_85] + | | | key expressions:_col0 (type: string), _col1 (type: string) + | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | | sort order:++ + | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | | Group By Operator [GBY_84] + | | | keys:_col0 (type: string), _col1 (type: string) + | | | outputColumnNames:["_col0","_col1"] + | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | | Select Operator [SEL_80] + | | | outputColumnNames:["_col0","_col1"] + | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | | Filter Operator [FIL_158] + | | | predicate:value is not null (type: boolean) + | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | | TableScan [TS_79] + | | | alias:y + | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | | |<-Map 29 [CONTAINS] + | | Reduce Output Operator [RS_94] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | sort order:++ | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_97] + | | Group By Operator [GBY_93] | | keys:_col0 (type: string), _col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_90] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 24 [SIMPLE_EDGE] - | | |<-Reducer 23 [CONTAINS] - | | | Reduce Output Operator [RS_89] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_88] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_81] - | | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | | outputColumnNames:["_col0","_col1"] - | | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | | |<-Union 22 [SIMPLE_EDGE] - | | | |<-Map 21 [CONTAINS] - | | | | Reduce Output Operator [RS_80] - | | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | | sort order:++ - | | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | | Group By Operator [GBY_79] - | | | | keys:_col0 (type: string), _col1 (type: string) - | | | | outputColumnNames:["_col0","_col1"] - | | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | | Select Operator [SEL_73] - | | | | outputColumnNames:["_col0","_col1"] - | | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | | Filter Operator [FIL_160] - | | | | predicate:value is not null (type: boolean) - | | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | | TableScan [TS_72] - | | | | alias:x - | | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | | |<-Map 28 [CONTAINS] - | | | Reduce Output Operator [RS_80] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_79] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_75] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_161] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_74] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 29 [CONTAINS] - | | Reduce Output Operator [RS_89] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_88] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_84] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_162] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_83] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_71] + | | Select Operator [SEL_89] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_159] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_88] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Select Operator [SEL_100] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_159] + | Filter Operator [FIL_161] | predicate:key is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_70] + | TableScan [TS_99] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 3 [CONTAINS] - Reduce Output Operator [RS_119] + |<-Reducer 8 [CONTAINS] + Reduce Output Operator [RS_116] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_118] + Group By Operator [GBY_115] keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_68] + Group By Operator [GBY_66] | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Union 2 [SIMPLE_EDGE] - |<-Map 11 [CONTAINS] - | Reduce Output Operator [RS_67] + |<-Union 7 [SIMPLE_EDGE] + |<-Map 19 [CONTAINS] + | Reduce Output Operator [RS_65] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_66] + | Group By Operator [GBY_64] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_62] + | Select Operator [SEL_60] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_168] + | Map Join Operator [MAPJOIN_165] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 11":"_col0 (type: string)","Reducer 16":"_col2 (type: string)"} - | | outputColumnNames:["_col1","_col4"] + | | keys:{"Map 19":"_col0 (type: string)","Reducer 15":"_col2 (type: string)"} + | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 16 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_60] + | |<-Reducer 15 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_56] | | key expressions:_col2 (type: string) | | Map-reduce partition columns:_col2 (type: string) | | sort order:+ | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | | Map Join Operator [MAPJOIN_167] + | | Map Join Operator [MAPJOIN_164] | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Map 19":"_col1 (type: string)","Reducer 16":"_col1 (type: string)"} + | | | keys:{"Reducer 15":"_col1 (type: string)","Map 18":"_col1 (type: string)"} | | | outputColumnNames:["_col2"] | | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 19 [BROADCAST_EDGE] - | | | Reduce Output Operator [RS_54] + | | |<-Map 18 [BROADCAST_EDGE] + | | | Reduce Output Operator [RS_53] | | | key expressions:_col1 (type: string) | | | Map-reduce partition columns:_col1 (type: string) | | | sort order:+ | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | | | value expressions:_col0 (type: string) - | | | Select Operator [SEL_50] + | | | Select Operator [SEL_47] | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_158] + | | | Filter Operator [FIL_154] | | | predicate:(value is not null and key is not null) (type: boolean) | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_49] + | | | TableScan [TS_46] | | | alias:x | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Select Operator [SEL_48] + | | |<-Select Operator [SEL_45] | | outputColumnNames:["_col1"] | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_47] + | | Group By Operator [GBY_44] | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 15 [SIMPLE_EDGE] - | | |<-Reducer 14 [CONTAINS] - | | | Reduce Output Operator [RS_46] + | | |<-Union 14 [SIMPLE_EDGE] + | | |<-Reducer 13 [CONTAINS] + | | | Reduce Output Operator [RS_43] | | | key expressions:_col0 (type: string), _col1 (type: string) | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | | sort order:++ | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_45] + | | | Group By Operator [GBY_42] | | | keys:_col0 (type: string), _col1 (type: string) | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_38] + | | | Group By Operator [GBY_35] | | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | | | outputColumnNames:["_col0","_col1"] | | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | | |<-Union 13 [SIMPLE_EDGE] - | | | |<-Map 12 [CONTAINS] - | | | | Reduce Output Operator [RS_37] + | | | |<-Union 12 [SIMPLE_EDGE] + | | | |<-Map 11 [CONTAINS] + | | | | Reduce Output Operator [RS_34] | | | | key expressions:_col0 (type: string), _col1 (type: string) | | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | | | sort order:++ | | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | | Group By Operator [GBY_36] + | | | | Group By Operator [GBY_33] | | | | keys:_col0 (type: string), _col1 (type: string) | | | | outputColumnNames:["_col0","_col1"] | | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | | Select Operator [SEL_30] + | | | | Select Operator [SEL_27] | | | | outputColumnNames:["_col0","_col1"] | | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | | Filter Operator [FIL_155] + | | | | Filter Operator [FIL_151] | | | | predicate:value is not null (type: boolean) | | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | | TableScan [TS_29] + | | | | TableScan [TS_26] | | | | alias:x | | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | | |<-Map 17 [CONTAINS] - | | | Reduce Output Operator [RS_37] + | | | |<-Map 16 [CONTAINS] + | | | Reduce Output Operator [RS_34] | | | key expressions:_col0 (type: string), _col1 (type: string) | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | | sort order:++ | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_36] + | | | Group By Operator [GBY_33] | | | keys:_col0 (type: string), _col1 (type: string) | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_32] + | | | Select Operator [SEL_29] | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_156] + | | | Filter Operator [FIL_152] | | | predicate:value is not null (type: boolean) | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_31] + | | | TableScan [TS_28] | | | alias:y | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 18 [CONTAINS] - | | Reduce Output Operator [RS_46] + | | |<-Map 17 [CONTAINS] + | | Reduce Output Operator [RS_43] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | sort order:++ | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_45] + | | Group By Operator [GBY_42] | | keys:_col0 (type: string), _col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_41] + | | Select Operator [SEL_38] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_157] + | | Filter Operator [FIL_153] | | predicate:value is not null (type: boolean) | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_40] + | | TableScan [TS_37] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_28] + | |<-Select Operator [SEL_49] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_154] + | Filter Operator [FIL_155] | predicate:key is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_27] + | TableScan [TS_48] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 1 [CONTAINS] - Reduce Output Operator [RS_67] + |<-Map 6 [CONTAINS] + Reduce Output Operator [RS_65] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_66] + Group By Operator [GBY_64] keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_26] + Select Operator [SEL_25] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Map Join Operator [MAPJOIN_166] + Map Join Operator [MAPJOIN_163] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 1":"_col0 (type: string)","Reducer 8":"_col2 (type: string)"} - | outputColumnNames:["_col1","_col4"] + | keys:{"Reducer 3":"_col2 (type: string)","Map 6":"_col0 (type: string)"} + | outputColumnNames:["_col2","_col5"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 8 [BROADCAST_EDGE] - | Reduce Output Operator [RS_24] + |<-Reducer 3 [BROADCAST_EDGE] + | Reduce Output Operator [RS_21] | key expressions:_col2 (type: string) | Map-reduce partition columns:_col2 (type: string) | sort order:+ | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_165] + | Map Join Operator [MAPJOIN_162] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 10":"_col1 (type: string)","Reducer 8":"_col1 (type: string)"} + | | keys:{"Reducer 3":"_col1 (type: string)","Map 5":"_col1 (type: string)"} | | outputColumnNames:["_col2"] | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | |<-Map 10 [BROADCAST_EDGE] + | |<-Map 5 [BROADCAST_EDGE] | | Reduce Output Operator [RS_18] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string) - | | Select Operator [SEL_14] + | | Select Operator [SEL_12] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_153] + | | Filter Operator [FIL_149] | | predicate:(value is not null and key is not null) (type: boolean) | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_13] + | | TableScan [TS_11] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_12] + | |<-Select Operator [SEL_10] | outputColumnNames:["_col1"] | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_11] + | Group By Operator [GBY_9] | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 7 [SIMPLE_EDGE] - | |<-Map 6 [CONTAINS] - | | Reduce Output Operator [RS_10] + | |<-Union 2 [SIMPLE_EDGE] + | |<-Map 1 [CONTAINS] + | | Reduce Output Operator [RS_8] | | key expressions:_col0 (type: string), _col1 (type: string) | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | | sort order:++ | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_9] + | | Group By Operator [GBY_7] | | keys:_col0 (type: string), _col1 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_3] + | | Select Operator [SEL_1] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_151] + | | Filter Operator [FIL_147] | | predicate:value is not null (type: boolean) | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_2] + | | TableScan [TS_0] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Map 9 [CONTAINS] - | Reduce Output Operator [RS_10] + | |<-Map 4 [CONTAINS] + | Reduce Output Operator [RS_8] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_9] + | Group By Operator [GBY_7] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_5] + | Select Operator [SEL_3] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_152] + | Filter Operator [FIL_148] | predicate:value is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_4] + | TableScan [TS_2] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_1] + |<-Select Operator [SEL_14] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Filter Operator [FIL_150] predicate:key is not null (type: boolean) Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_0] + TableScan [TS_13] alias:y Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -3027,370 +3027,370 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage +Map 1 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) Map 12 <- Union 9 (CONTAINS) Map 13 <- Union 9 (CONTAINS) -Map 16 <- Map 21 (BROADCAST_EDGE), Union 3 (CONTAINS) -Map 17 <- Map 21 (BROADCAST_EDGE), Union 3 (CONTAINS) -Map 18 <- Map 21 (BROADCAST_EDGE), Union 3 (CONTAINS) -Map 19 <- Map 21 (BROADCAST_EDGE), Union 3 (CONTAINS) -Map 21 <- Map 20 (BROADCAST_EDGE) -Map 4 <- Map 7 (BROADCAST_EDGE), Union 5 (CONTAINS) -Map 6 <- Map 7 (BROADCAST_EDGE), Union 5 (CONTAINS) +Map 17 <- Map 16 (BROADCAST_EDGE) +Map 18 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) +Map 19 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) +Map 20 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) +Map 21 <- Map 17 (BROADCAST_EDGE), Union 4 (CONTAINS) +Map 5 <- Map 6 (BROADCAST_EDGE), Union 2 (CONTAINS) Map 8 <- Union 9 (CONTAINS) Reducer 10 <- Map 14 (SIMPLE_EDGE), Union 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS), Union 5 (SIMPLE_EDGE) +Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 4 (CONTAINS) +Reducer 3 <- Map 7 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) Stage-0 Fetch Operator limit:-1 Stage-1 - Union 3 + Union 4 + |<-Reducer 3 [CONTAINS] + | File Output Operator [FS_75] + | compressed:false + | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE + | table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} + | Select Operator [SEL_20] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 317 Data size: 3333 Basic stats: COMPLETE Column stats: NONE + | Merge Join Operator [MERGEJOIN_116] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"1":"_col0 (type: string)","0":"_col1 (type: string)"} + | | outputColumnNames:["_col1","_col4"] + | | Statistics:Num rows: 317 Data size: 3333 Basic stats: COMPLETE Column stats: NONE + | |<-Map 7 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_18] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) + | | sort order:+ + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col1 (type: string) + | | Select Operator [SEL_9] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_103] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_8] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Union 2 [SIMPLE_EDGE] + | |<-Map 1 [CONTAINS] + | | Reduce Output Operator [RS_16] + | | key expressions:_col1 (type: string) + | | Map-reduce partition columns:_col1 (type: string) + | | sort order:+ + | | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE + | | Map Join Operator [MAPJOIN_115] + | | | condition map:[{"":"Inner Join 0 to 1"}] + | | | keys:{"Map 1":"_col0 (type: string)","Map 6":"_col1 (type: string)"} + | | | outputColumnNames:["_col1"] + | | | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE + | | |<-Map 6 [BROADCAST_EDGE] + | | | Reduce Output Operator [RS_13] + | | | key expressions:_col1 (type: string) + | | | Map-reduce partition columns:_col1 (type: string) + | | | sort order:+ + | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | | value expressions:_col0 (type: string) + | | | Select Operator [SEL_7] + | | | outputColumnNames:["_col0","_col1"] + | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | | Filter Operator [FIL_102] + | | | predicate:(value is not null and key is not null) (type: boolean) + | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | | TableScan [TS_6] + | | | alias:x + | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | | | Reduce Output Operator [RS_121] + | | | key expressions:_col1 (type: string) + | | | Map-reduce partition columns:_col1 (type: string) + | | | sort order:+ + | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | | value expressions:_col0 (type: string) + | | | Please refer to the previous Select Operator [SEL_7] + | | |<-Select Operator [SEL_1] + | | outputColumnNames:["_col0"] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_100] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_0] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Map 5 [CONTAINS] + | Reduce Output Operator [RS_16] + | key expressions:_col1 (type: string) + | Map-reduce partition columns:_col1 (type: string) + | sort order:+ + | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE + | Map Join Operator [MAPJOIN_115] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Map 5":"_col0 (type: string)","Map 6":"_col1 (type: string)"} + | | outputColumnNames:["_col1"] + | | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE + | |<- Please refer to the previous Map 6 [BROADCAST_EDGE] + | |<-Select Operator [SEL_3] + | outputColumnNames:["_col0"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_101] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_2] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE |<-Reducer 11 [CONTAINS] - | File Output Operator [FS_77] + | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE | table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - | Select Operator [SEL_45] + | Select Operator [SEL_44] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 620 Data size: 6547 Basic stats: COMPLETE Column stats: NONE - | Merge Join Operator [MERGEJOIN_120] + | Merge Join Operator [MERGEJOIN_118] | | condition map:[{"":"Inner Join 0 to 1"}] | | keys:{"1":"_col0 (type: string)","0":"_col1 (type: string)"} | | outputColumnNames:["_col1","_col4"] | | Statistics:Num rows: 620 Data size: 6547 Basic stats: COMPLETE Column stats: NONE | |<-Map 15 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_43] + | | Reduce Output Operator [RS_42] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col1 (type: string) - | | Select Operator [SEL_34] + | | Select Operator [SEL_33] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_110] + | | Filter Operator [FIL_108] | | predicate:key is not null (type: boolean) | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_33] + | | TableScan [TS_32] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE | |<-Reducer 10 [SIMPLE_EDGE] - | Reduce Output Operator [RS_41] + | Reduce Output Operator [RS_40] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) | sort order:+ | Statistics:Num rows: 564 Data size: 5952 Basic stats: COMPLETE Column stats: NONE - | Merge Join Operator [MERGEJOIN_119] + | Merge Join Operator [MERGEJOIN_117] | | condition map:[{"":"Inner Join 0 to 1"}] | | keys:{"1":"_col1 (type: string)","0":"_col0 (type: string)"} | | outputColumnNames:["_col1"] | | Statistics:Num rows: 564 Data size: 5952 Basic stats: COMPLETE Column stats: NONE | |<-Map 14 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_38] + | | Reduce Output Operator [RS_37] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string) - | | Select Operator [SEL_32] + | | Select Operator [SEL_31] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_109] + | | Filter Operator [FIL_107] | | predicate:(value is not null and key is not null) (type: boolean) | | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_31] + | | TableScan [TS_30] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE | |<-Union 9 [SIMPLE_EDGE] | |<-Map 12 [CONTAINS] - | | Reduce Output Operator [RS_36] + | | Reduce Output Operator [RS_35] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ | | Statistics:Num rows: 513 Data size: 5411 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_25] + | | Select Operator [SEL_24] | | outputColumnNames:["_col0"] | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_107] + | | Filter Operator [FIL_105] | | predicate:value is not null (type: boolean) | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_24] + | | TableScan [TS_23] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE | |<-Map 13 [CONTAINS] - | | Reduce Output Operator [RS_36] + | | Reduce Output Operator [RS_35] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ | | Statistics:Num rows: 513 Data size: 5411 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_29] + | | Select Operator [SEL_28] | | outputColumnNames:["_col0"] | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_108] + | | Filter Operator [FIL_106] | | predicate:value is not null (type: boolean) | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_28] + | | TableScan [TS_27] | | alias:y | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE | |<-Map 8 [CONTAINS] - | Reduce Output Operator [RS_36] + | Reduce Output Operator [RS_35] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 513 Data size: 5411 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_23] + | Select Operator [SEL_22] | outputColumnNames:["_col0"] | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_106] + | Filter Operator [FIL_104] | predicate:value is not null (type: boolean) | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_22] + | TableScan [TS_21] | alias:x | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map 19 [CONTAINS] - | File Output Operator [FS_77] + |<-Map 21 [CONTAINS] + | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE | table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - | Select Operator [SEL_75] + | Select Operator [SEL_73] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_122] + | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 21":"_col1 (type: string)","Map 19":"_col0 (type: string)"} - | | outputColumnNames:["_col1","_col4"] + | | keys:{"Map 21":"_col0 (type: string)","Map 17":"_col1 (type: string)"} + | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<-Map 21 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_73] + | |<-Map 17 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_69] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string), _col3 (type: string) - | | Map Join Operator [MAPJOIN_121] + | | Map Join Operator [MAPJOIN_119] | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"Map 21":"_col0 (type: string)","Map 20":"_col0 (type: string)"} + | | | keys:{"Map 16":"_col0 (type: string)","Map 17":"_col0 (type: string)"} | | | outputColumnNames:["_col0","_col1","_col3"] | | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 20 [BROADCAST_EDGE] - | | | Reduce Output Operator [RS_65] + | | |<-Map 16 [BROADCAST_EDGE] + | | | Reduce Output Operator [RS_64] | | | key expressions:_col0 (type: string) | | | Map-reduce partition columns:_col0 (type: string) | | | sort order:+ | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE | | | value expressions:_col1 (type: string) - | | | Select Operator [SEL_61] + | | | Select Operator [SEL_48] | | | outputColumnNames:["_col0","_col1"] | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_115] + | | | Filter Operator [FIL_109] | | | predicate:(key is not null and value is not null) (type: boolean) | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_60] + | | | TableScan [TS_47] | | | alias:x | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | |<-Select Operator [SEL_63] + | | |<-Select Operator [SEL_50] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_116] + | | Filter Operator [FIL_110] | | predicate:key is not null (type: boolean) | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_62] + | | TableScan [TS_49] | | alias:x | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | Reduce Output Operator [RS_126] + | | Reduce Output Operator [RS_125] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string), _col3 (type: string) - | | Please refer to the previous Map Join Operator [MAPJOIN_121] - | | Reduce Output Operator [RS_127] + | | Please refer to the previous Map Join Operator [MAPJOIN_119] + | | Reduce Output Operator [RS_126] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string), _col3 (type: string) - | | Please refer to the previous Map Join Operator [MAPJOIN_121] - | | Reduce Output Operator [RS_128] + | | Please refer to the previous Map Join Operator [MAPJOIN_119] + | | Reduce Output Operator [RS_127] | | key expressions:_col1 (type: string) | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col0 (type: string), _col3 (type: string) - | | Please refer to the previous Map Join Operator [MAPJOIN_121] - | |<-Select Operator [SEL_58] + | | Please refer to the previous Map Join Operator [MAPJOIN_119] + | |<-Select Operator [SEL_61] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE | Filter Operator [FIL_114] | predicate:value is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_57] + | TableScan [TS_60] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 16 [CONTAINS] - | File Output Operator [FS_77] + |<-Map 20 [CONTAINS] + | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE | table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - | Select Operator [SEL_75] + | Select Operator [SEL_73] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_122] + | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 21":"_col1 (type: string)","Map 16":"_col0 (type: string)"} - | | outputColumnNames:["_col1","_col4"] + | | keys:{"Map 20":"_col0 (type: string)","Map 17":"_col1 (type: string)"} + | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<- Please refer to the previous Map 21 [BROADCAST_EDGE] - | |<-Select Operator [SEL_49] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_111] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_48] - | alias:x - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map 18 [CONTAINS] - | File Output Operator [FS_77] - | compressed:false - | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE - | table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - | Select Operator [SEL_75] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_122] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 21":"_col1 (type: string)","Map 18":"_col0 (type: string)"} - | | outputColumnNames:["_col1","_col4"] - | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<- Please refer to the previous Map 21 [BROADCAST_EDGE] - | |<-Select Operator [SEL_55] + | |<- Please refer to the previous Map 17 [BROADCAST_EDGE] + | |<-Select Operator [SEL_58] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE | Filter Operator [FIL_113] | predicate:value is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_54] + | TableScan [TS_57] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Map 17 [CONTAINS] - | File Output Operator [FS_77] + |<-Map 19 [CONTAINS] + | File Output Operator [FS_75] | compressed:false | Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE | table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - | Select Operator [SEL_75] + | Select Operator [SEL_73] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_122] + | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 21":"_col1 (type: string)","Map 17":"_col0 (type: string)"} - | | outputColumnNames:["_col1","_col4"] + | | keys:{"Map 19":"_col0 (type: string)","Map 17":"_col1 (type: string)"} + | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE - | |<- Please refer to the previous Map 21 [BROADCAST_EDGE] - | |<-Select Operator [SEL_51] + | |<- Please refer to the previous Map 17 [BROADCAST_EDGE] + | |<-Select Operator [SEL_54] | outputColumnNames:["_col0"] | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE | Filter Operator [FIL_112] | predicate:value is not null (type: boolean) | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_50] + | TableScan [TS_53] | alias:y | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 2 [CONTAINS] - File Output Operator [FS_77] + |<-Map 18 [CONTAINS] + File Output Operator [FS_75] compressed:false Statistics:Num rows: 1776 Data size: 18753 Basic stats: COMPLETE Column stats: NONE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Select Operator [SEL_21] + Select Operator [SEL_73] outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 317 Data size: 3333 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_118] + Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE + Map Join Operator [MAPJOIN_120] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"1":"_col1 (type: string)","0":"_col0 (type: string)"} - | outputColumnNames:["_col1","_col3"] - | Statistics:Num rows: 317 Data size: 3333 Basic stats: COMPLETE Column stats: NONE - |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_17] - | key expressions:_col0 (type: string) - | Map-reduce partition columns:_col0 (type: string) - | sort order:+ - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | value expressions:_col1 (type: string) - | Select Operator [SEL_1] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_102] - | predicate:key is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_0] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Union 5 [SIMPLE_EDGE] - |<-Map 4 [CONTAINS] - | Reduce Output Operator [RS_19] - | key expressions:_col1 (type: string) - | Map-reduce partition columns:_col1 (type: string) - | sort order:+ - | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_117] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 4":"_col0 (type: string)","Map 7":"_col1 (type: string)"} - | | outputColumnNames:["_col1"] - | | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - | |<-Map 7 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_13] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col0 (type: string) - | | Select Operator [SEL_9] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_105] - | | predicate:(value is not null and key is not null) (type: boolean) - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_8] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | Reduce Output Operator [RS_123] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col0 (type: string) - | | Please refer to the previous Select Operator [SEL_9] - | |<-Select Operator [SEL_3] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_103] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_2] - | alias:x - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map 6 [CONTAINS] - Reduce Output Operator [RS_19] - key expressions:_col1 (type: string) - Map-reduce partition columns:_col1 (type: string) - sort order:+ - Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - Map Join Operator [MAPJOIN_117] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 7":"_col1 (type: string)","Map 6":"_col0 (type: string)"} - | outputColumnNames:["_col1"] - | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE - |<- Please refer to the previous Map 7 [BROADCAST_EDGE] - |<-Select Operator [SEL_5] - outputColumnNames:["_col0"] - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_104] - predicate:value is not null (type: boolean) - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_4] - alias:y - Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | keys:{"Map 18":"_col0 (type: string)","Map 17":"_col1 (type: string)"} + | outputColumnNames:["_col0","_col3"] + | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE + |<- Please refer to the previous Map 17 [BROADCAST_EDGE] + |<-Select Operator [SEL_52] + outputColumnNames:["_col0"] + Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_111] + predicate:value is not null (type: boolean) + Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_51] + alias:x + Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: explain SELECT x.key, y.value @@ -3421,480 +3421,480 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Map 10 <- Union 8 (CONTAINS) +Map 1 <- Union 2 (CONTAINS) Map 12 <- Union 13 (CONTAINS) Map 19 <- Union 13 (CONTAINS) Map 20 <- Union 15 (CONTAINS) -Map 23 <- Union 24 (CONTAINS) -Map 30 <- Union 24 (CONTAINS) -Map 31 <- Union 26 (CONTAINS) -Map 32 <- Union 28 (CONTAINS) -Map 34 <- Map 33 (BROADCAST_EDGE) -Map 7 <- Union 8 (CONTAINS) +Map 24 <- Map 23 (BROADCAST_EDGE) +Map 25 <- Union 26 (CONTAINS) +Map 32 <- Union 26 (CONTAINS) +Map 33 <- Union 28 (CONTAINS) +Map 34 <- Union 30 (CONTAINS) +Map 9 <- Union 2 (CONTAINS) Reducer 14 <- Union 13 (SIMPLE_EDGE), Union 15 (CONTAINS) Reducer 16 <- Union 15 (SIMPLE_EDGE) Reducer 17 <- Map 21 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE), Union 3 (CONTAINS) -Reducer 25 <- Union 24 (SIMPLE_EDGE), Union 26 (CONTAINS) +Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 27 <- Union 26 (SIMPLE_EDGE), Union 28 (CONTAINS) -Reducer 29 <- Map 34 (BROADCAST_EDGE), Union 28 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 6 <- Union 5 (SIMPLE_EDGE) -Reducer 9 <- Map 11 (BROADCAST_EDGE), Union 8 (SIMPLE_EDGE) +Reducer 29 <- Union 28 (SIMPLE_EDGE), Union 30 (CONTAINS) +Reducer 3 <- Map 10 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE) +Reducer 31 <- Map 24 (BROADCAST_EDGE), Union 30 (SIMPLE_EDGE), Union 7 (CONTAINS) +Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) +Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) +Reducer 8 <- Union 7 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 6 - File Output Operator [FS_121] + Reducer 8 + File Output Operator [FS_119] compressed:false Statistics:Num rows: 258 Data size: 2737 Basic stats: COMPLETE Column stats: NONE table:{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"} - Group By Operator [GBY_119] + Group By Operator [GBY_117] | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 258 Data size: 2737 Basic stats: COMPLETE Column stats: NONE - |<-Union 5 [SIMPLE_EDGE] - |<-Reducer 4 [CONTAINS] - | Reduce Output Operator [RS_118] + |<-Union 7 [SIMPLE_EDGE] + |<-Reducer 31 [CONTAINS] + | Reduce Output Operator [RS_116] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 517 Data size: 5486 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_117] + | Group By Operator [GBY_115] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 517 Data size: 5486 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_67] - | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Union 3 [SIMPLE_EDGE] - | |<-Reducer 18 [CONTAINS] - | | Reduce Output Operator [RS_66] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_65] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_61] + | Select Operator [SEL_111] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE + | Map Join Operator [MAPJOIN_164] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Reducer 31":"_col1 (type: string)","Map 24":"_col1 (type: string)"} + | | outputColumnNames:["_col0","_col3"] + | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE + | |<-Map 24 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_107] + | | key expressions:_col1 (type: string) + | | Map-reduce partition columns:_col1 (type: string) + | | sort order:+ + | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col0 (type: string), _col3 (type: string) + | | Map Join Operator [MAPJOIN_163] + | | | condition map:[{"":"Inner Join 0 to 1"}] + | | | keys:{"Map 24":"_col0 (type: string)","Map 23":"_col0 (type: string)"} + | | | outputColumnNames:["_col0","_col1","_col3"] + | | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE + | | |<-Map 23 [BROADCAST_EDGE] + | | | Reduce Output Operator [RS_102] + | | | key expressions:_col0 (type: string) + | | | Map-reduce partition columns:_col0 (type: string) + | | | sort order:+ + | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | | value expressions:_col1 (type: string) + | | | Select Operator [SEL_69] + | | | outputColumnNames:["_col0","_col1"] + | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | | Filter Operator [FIL_153] + | | | predicate:(key is not null and value is not null) (type: boolean) + | | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | | TableScan [TS_68] + | | | alias:x + | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | | |<-Select Operator [SEL_71] | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | | Merge Join Operator [MERGEJOIN_164] - | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"1":"_col0 (type: string)","0":"_col2 (type: string)"} - | | | outputColumnNames:["_col2","_col5"] - | | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 22 [SIMPLE_EDGE] - | | | Reduce Output Operator [RS_59] - | | | key expressions:_col0 (type: string) - | | | Map-reduce partition columns:_col0 (type: string) - | | | sort order:+ - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | value expressions:_col1 (type: string) - | | | Select Operator [SEL_50] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_154] - | | | predicate:key is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_49] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Reducer 17 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_57] - | | key expressions:_col2 (type: string) - | | Map-reduce partition columns:_col2 (type: string) - | | sort order:+ - | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | | Merge Join Operator [MERGEJOIN_163] - | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"1":"_col1 (type: string)","0":"_col1 (type: string)"} - | | | outputColumnNames:["_col2"] - | | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 21 [SIMPLE_EDGE] - | | | Reduce Output Operator [RS_54] - | | | key expressions:_col1 (type: string) - | | | Map-reduce partition columns:_col1 (type: string) - | | | sort order:+ - | | | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | | | value expressions:_col0 (type: string) - | | | Select Operator [SEL_48] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_153] - | | | predicate:(value is not null and key is not null) (type: boolean) - | | | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_47] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Reducer 16 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_52] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_46] - | | outputColumnNames:["_col1"] - | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_45] - | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - | | |<-Union 15 [SIMPLE_EDGE] - | | |<-Reducer 14 [CONTAINS] - | | | Reduce Output Operator [RS_44] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_43] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_36] - | | | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | | | outputColumnNames:["_col0","_col1"] - | | | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | | | |<-Union 13 [SIMPLE_EDGE] - | | | |<-Map 12 [CONTAINS] - | | | | Reduce Output Operator [RS_35] - | | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | | sort order:++ - | | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | | Group By Operator [GBY_34] - | | | | keys:_col0 (type: string), _col1 (type: string) - | | | | outputColumnNames:["_col0","_col1"] - | | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | | Select Operator [SEL_28] - | | | | outputColumnNames:["_col0","_col1"] - | | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | | Filter Operator [FIL_150] - | | | | predicate:value is not null (type: boolean) - | | | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | | | | TableScan [TS_27] - | | | | alias:x - | | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | | | |<-Map 19 [CONTAINS] - | | | Reduce Output Operator [RS_35] - | | | key expressions:_col0 (type: string), _col1 (type: string) - | | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | | sort order:++ - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Group By Operator [GBY_34] - | | | keys:_col0 (type: string), _col1 (type: string) - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | | Select Operator [SEL_30] - | | | outputColumnNames:["_col0","_col1"] - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | Filter Operator [FIL_151] - | | | predicate:value is not null (type: boolean) - | | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | | TableScan [TS_29] - | | | alias:y - | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | | |<-Map 20 [CONTAINS] - | | Reduce Output Operator [RS_44] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_43] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_39] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_152] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_38] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 2 [CONTAINS] - | Reduce Output Operator [RS_66] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_65] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_26] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | Merge Join Operator [MERGEJOIN_162] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"1":"_col2 (type: string)","0":"_col0 (type: string)"} - | | outputColumnNames:["_col1","_col4"] - | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - | |<-Map 1 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_22] - | | key expressions:_col0 (type: string) - | | Map-reduce partition columns:_col0 (type: string) - | | sort order:+ - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col1 (type: string) - | | Select Operator [SEL_1] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_146] - | | predicate:key is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_0] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Reducer 9 [SIMPLE_EDGE] - | Reduce Output Operator [RS_24] - | key expressions:_col2 (type: string) - | Map-reduce partition columns:_col2 (type: string) - | sort order:+ - | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | Map Join Operator [MAPJOIN_161] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 11":"_col1 (type: string)","Reducer 9":"_col1 (type: string)"} - | | outputColumnNames:["_col2"] - | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE - | |<-Map 11 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_18] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col0 (type: string) - | | Select Operator [SEL_14] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_149] - | | predicate:(value is not null and key is not null) (type: boolean) - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_13] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_12] - | outputColumnNames:["_col1"] - | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_11] - | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 8 [SIMPLE_EDGE] - | |<-Map 10 [CONTAINS] - | | Reduce Output Operator [RS_10] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_9] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_5] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_148] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_4] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Map 7 [CONTAINS] - | Reduce Output Operator [RS_10] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_9] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_3] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_147] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_2] - | alias:x - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_118] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_154] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_70] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Select Operator [SEL_100] + | outputColumnNames:["_col1"] + | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_99] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE + | |<-Union 30 [SIMPLE_EDGE] + | |<-Map 34 [CONTAINS] + | | Reduce Output Operator [RS_98] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_97] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_93] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_158] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_92] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 29 [CONTAINS] + | Reduce Output Operator [RS_98] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_97] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_90] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + | |<-Union 28 [SIMPLE_EDGE] + | |<-Map 33 [CONTAINS] + | | Reduce Output Operator [RS_89] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_88] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_84] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_157] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_83] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 27 [CONTAINS] + | Reduce Output Operator [RS_89] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_88] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_81] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | |<-Union 26 [SIMPLE_EDGE] + | |<-Map 25 [CONTAINS] + | | Reduce Output Operator [RS_80] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_79] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_73] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_155] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_72] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Map 32 [CONTAINS] + | Reduce Output Operator [RS_80] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_79] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_75] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_156] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_74] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 6 [CONTAINS] + Reduce Output Operator [RS_116] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 517 Data size: 5486 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_117] + Group By Operator [GBY_115] keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 517 Data size: 5486 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_113] - outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - Map Join Operator [MAPJOIN_166] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"Map 34":"_col1 (type: string)","Reducer 29":"_col1 (type: string)"} - | outputColumnNames:["_col2","_col5"] - | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE - |<-Map 34 [BROADCAST_EDGE] - | Reduce Output Operator [RS_111] - | key expressions:_col1 (type: string) - | Map-reduce partition columns:_col1 (type: string) - | sort order:+ - | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE - | value expressions:_col0 (type: string), _col3 (type: string) - | Map Join Operator [MAPJOIN_165] - | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"Map 34":"_col0 (type: string)","Map 33":"_col0 (type: string)"} - | | outputColumnNames:["_col0","_col1","_col3"] - | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE - | |<-Map 33 [BROADCAST_EDGE] - | | Reduce Output Operator [RS_103] - | | key expressions:_col0 (type: string) - | | Map-reduce partition columns:_col0 (type: string) - | | sort order:+ - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col1 (type: string) - | | Select Operator [SEL_99] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_159] - | | predicate:(key is not null and value is not null) (type: boolean) - | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_98] - | | alias:x - | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - | |<-Select Operator [SEL_101] + Group By Operator [GBY_66] + | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + |<-Union 5 [SIMPLE_EDGE] + |<-Reducer 4 [CONTAINS] + | Reduce Output Operator [RS_65] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_64] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_25] | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_160] - | predicate:key is not null (type: boolean) - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_100] - | alias:x - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Select Operator [SEL_97] - outputColumnNames:["_col1"] - Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_96] - | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 220 Data size: 2332 Basic stats: COMPLETE Column stats: NONE - |<-Union 28 [SIMPLE_EDGE] - |<-Map 32 [CONTAINS] - | Reduce Output Operator [RS_95] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_94] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_90] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_158] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_89] - | alias:y - | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 27 [CONTAINS] - Reduce Output Operator [RS_95] - key expressions:_col0 (type: string), _col1 (type: string) - Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - sort order:++ - Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_94] - keys:_col0 (type: string), _col1 (type: string) - outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 440 Data size: 4664 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_87] - | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE - |<-Union 26 [SIMPLE_EDGE] - |<-Reducer 25 [CONTAINS] - | Reduce Output Operator [RS_86] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_85] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_78] - | | keys:KEY._col0 (type: string), KEY._col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE - | |<-Union 24 [SIMPLE_EDGE] - | |<-Map 30 [CONTAINS] - | | Reduce Output Operator [RS_77] - | | key expressions:_col0 (type: string), _col1 (type: string) - | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | | sort order:++ - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Group By Operator [GBY_76] - | | keys:_col0 (type: string), _col1 (type: string) - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | | Select Operator [SEL_72] - | | outputColumnNames:["_col0","_col1"] - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_156] - | | predicate:value is not null (type: boolean) - | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_71] - | | alias:y - | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - | |<-Map 23 [CONTAINS] - | Reduce Output Operator [RS_77] - | key expressions:_col0 (type: string), _col1 (type: string) - | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - | sort order:++ - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_76] - | keys:_col0 (type: string), _col1 (type: string) - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_70] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_155] - | predicate:value is not null (type: boolean) - | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_69] - | alias:x - | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - |<-Map 31 [CONTAINS] - Reduce Output Operator [RS_86] - key expressions:_col0 (type: string), _col1 (type: string) - Map-reduce partition columns:_col0 (type: string), _col1 (type: string) - sort order:++ - Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_85] - keys:_col0 (type: string), _col1 (type: string) - outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_81] - outputColumnNames:["_col0","_col1"] - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_157] - predicate:value is not null (type: boolean) - Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_80] - alias:y - Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + | Merge Join Operator [MERGEJOIN_160] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"1":"_col0 (type: string)","0":"_col2 (type: string)"} + | | outputColumnNames:["_col2","_col5"] + | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + | |<-Map 11 [SIMPLE_EDGE] + | | Reduce Output Operator [RS_23] + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) + | | sort order:+ + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col1 (type: string) + | | Select Operator [SEL_14] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_147] + | | predicate:key is not null (type: boolean) + | | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_13] + | | alias:y + | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + | |<-Reducer 3 [SIMPLE_EDGE] + | Reduce Output Operator [RS_21] + | key expressions:_col2 (type: string) + | Map-reduce partition columns:_col2 (type: string) + | sort order:+ + | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE + | Map Join Operator [MAPJOIN_159] + | | condition map:[{"":"Inner Join 0 to 1"}] + | | keys:{"Reducer 3":"_col1 (type: string)","Map 10":"_col1 (type: string)"} + | | outputColumnNames:["_col2"] + | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE + | |<-Map 10 [BROADCAST_EDGE] + | | Reduce Output Operator [RS_18] + | | key expressions:_col1 (type: string) + | | Map-reduce partition columns:_col1 (type: string) + | | sort order:+ + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | value expressions:_col0 (type: string) + | | Select Operator [SEL_12] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_146] + | | predicate:(value is not null and key is not null) (type: boolean) + | | Statistics:Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_11] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Select Operator [SEL_10] + | outputColumnNames:["_col1"] + | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_9] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | |<-Union 2 [SIMPLE_EDGE] + | |<-Map 1 [CONTAINS] + | | Reduce Output Operator [RS_8] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_7] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_1] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_144] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_0] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Map 9 [CONTAINS] + | Reduce Output Operator [RS_8] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_7] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_3] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_145] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_2] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 18 [CONTAINS] + Reduce Output Operator [RS_65] + key expressions:_col0 (type: string), _col1 (type: string) + Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + sort order:++ + Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_64] + keys:_col0 (type: string), _col1 (type: string) + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 550 Data size: 5842 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_60] + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Merge Join Operator [MERGEJOIN_162] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"1":"_col0 (type: string)","0":"_col2 (type: string)"} + | outputColumnNames:["_col2","_col5"] + | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + |<-Map 22 [SIMPLE_EDGE] + | Reduce Output Operator [RS_58] + | key expressions:_col0 (type: string) + | Map-reduce partition columns:_col0 (type: string) + | sort order:+ + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col1 (type: string) + | Select Operator [SEL_49] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_152] + | predicate:key is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_48] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 17 [SIMPLE_EDGE] + Reduce Output Operator [RS_56] + key expressions:_col2 (type: string) + Map-reduce partition columns:_col2 (type: string) + sort order:+ + Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE + Merge Join Operator [MERGEJOIN_161] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"1":"_col1 (type: string)","0":"_col1 (type: string)"} + | outputColumnNames:["_col2"] + | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE + |<-Map 21 [SIMPLE_EDGE] + | Reduce Output Operator [RS_53] + | key expressions:_col1 (type: string) + | Map-reduce partition columns:_col1 (type: string) + | sort order:+ + | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col0 (type: string) + | Select Operator [SEL_47] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_151] + | predicate:(value is not null and key is not null) (type: boolean) + | Statistics:Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_46] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 16 [SIMPLE_EDGE] + Reduce Output Operator [RS_51] + key expressions:_col1 (type: string) + Map-reduce partition columns:_col1 (type: string) + sort order:+ + Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_45] + outputColumnNames:["_col1"] + Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_44] + | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 190 Data size: 2008 Basic stats: COMPLETE Column stats: NONE + |<-Union 15 [SIMPLE_EDGE] + |<-Reducer 14 [CONTAINS] + | Reduce Output Operator [RS_43] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_42] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_35] + | | keys:KEY._col0 (type: string), KEY._col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 131 Data size: 1372 Basic stats: COMPLETE Column stats: NONE + | |<-Union 13 [SIMPLE_EDGE] + | |<-Map 12 [CONTAINS] + | | Reduce Output Operator [RS_34] + | | key expressions:_col0 (type: string), _col1 (type: string) + | | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | | sort order:++ + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Group By Operator [GBY_33] + | | keys:_col0 (type: string), _col1 (type: string) + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | | Select Operator [SEL_27] + | | outputColumnNames:["_col0","_col1"] + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | Filter Operator [FIL_148] + | | predicate:value is not null (type: boolean) + | | Statistics:Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + | | TableScan [TS_26] + | | alias:x + | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + | |<-Map 19 [CONTAINS] + | Reduce Output Operator [RS_34] + | key expressions:_col0 (type: string), _col1 (type: string) + | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + | sort order:++ + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Group By Operator [GBY_33] + | keys:_col0 (type: string), _col1 (type: string) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 263 Data size: 2755 Basic stats: COMPLETE Column stats: NONE + | Select Operator [SEL_29] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_149] + | predicate:value is not null (type: boolean) + | Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_28] + | alias:y + | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + |<-Map 20 [CONTAINS] + Reduce Output Operator [RS_43] + key expressions:_col0 (type: string), _col1 (type: string) + Map-reduce partition columns:_col0 (type: string), _col1 (type: string) + sort order:++ + Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_42] + keys:_col0 (type: string), _col1 (type: string) + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 381 Data size: 4028 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_38] + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_150] + predicate:value is not null (type: boolean) + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_37] + alias:y + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: CREATE TABLE a(key STRING, value STRING) STORED AS TEXTFILE PREHOOK: type: CREATETABLE diff --git ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out index f2610de..23df010 100644 --- ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out +++ ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out @@ -63,82 +63,12 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: value is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - tag: 0 - auto parallelism: true - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: src - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - COLUMN_STATS_ACCURATE true - bucket_count -1 - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.src - numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src - Truncated Path -> Alias: - /src [src] - Map 2 - Map Operator Tree: - TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE GatherStats: false @@ -153,54 +83,50 @@ STAGE PLANS: Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 13 + Estimated key counts: Map 2 => 13 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Map 2 Position of Big Table: 0 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 250 + Estimated key counts: Map 3 => 250 keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 input vertices: - 0 Map 1 - Position of Big Table: 1 + 1 Map 3 + Position of Big Table: 0 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 1 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0 - columns.types string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -393,7 +319,7 @@ STAGE PLANS: /srcpart/ds=2008-04-08/hr=12 [srcpart] /srcpart/ds=2008-04-09/hr=11 [srcpart] /srcpart/ds=2008-04-09/hr=12 [srcpart] - Map 3 + Map 2 Map Operator Tree: TableScan alias: src1 @@ -463,6 +389,76 @@ STAGE PLANS: name: default.src1 Truncated Path -> Alias: /src1 [src1] + Map 3 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [src] Stage: Stage-0 Fetch Operator @@ -484,29 +480,12 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (value > 'val_450') (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Map 2 - Map Operator Tree: - TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -524,32 +503,28 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1 input vertices: - 1 Map 3 + 1 Map 2 Statistics: Num rows: 183 Data size: 1951 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col1 + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 input vertices: - 0 Map 1 + 1 Map 3 Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col0 + File Output Operator + compressed: false Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map 3 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 2 Map Operator Tree: TableScan alias: src1 @@ -566,6 +541,23 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value > 'val_450') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator