diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 7f79995f75..121f9bf083 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -769,6 +769,7 @@ minillaplocal.query.files=\ tez_union_multiinsert.q,\ tez_vector_dynpart_hashjoin_1.q,\ tez_vector_dynpart_hashjoin_2.q,\ + transitive_not_null.q,\ truncate_external_force.q,\ uber_reduce.q,\ udaf_collect_set_2.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java index 9711625016..48e2cc6781 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java @@ -26,7 +26,6 @@ import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.RelFactories.FilterFactory; import org.apache.calcite.rex.RexBuilder; @@ -47,10 +46,10 @@ public final class HiveJoinAddNotNullRule extends RelOptRule { public static final HiveJoinAddNotNullRule INSTANCE_JOIN = - new HiveJoinAddNotNullRule(HiveJoin.class, HiveRelFactories.HIVE_FILTER_FACTORY); + new HiveJoinAddNotNullRule(HiveJoin.class, HiveRelFactories.HIVE_FILTER_FACTORY); public static final HiveJoinAddNotNullRule INSTANCE_SEMIJOIN = - new HiveJoinAddNotNullRule(HiveSemiJoin.class, HiveRelFactories.HIVE_FILTER_FACTORY); + new HiveJoinAddNotNullRule(HiveSemiJoin.class, HiveRelFactories.HIVE_FILTER_FACTORY); private final FilterFactory filterFactory; @@ -60,7 +59,7 @@ * Creates an HiveJoinAddNotNullRule. */ public HiveJoinAddNotNullRule(Class clazz, - RelFactories.FilterFactory filterFactory) { + RelFactories.FilterFactory filterFactory) { super(operand(clazz, any())); this.filterFactory = filterFactory; } @@ -76,11 +75,27 @@ public void onMatch(RelOptRuleCall call) { HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); assert registry != null; - if (join.getJoinType() != JoinRelType.INNER) { + + if (join.getCondition().isAlwaysTrue()) { return; } - if (join.getCondition().isAlwaysTrue()) { + boolean genPredOnLeftSide = false, genPredOnRightSide = false; + + switch (join.getJoinType()) { + case INNER: + genPredOnLeftSide = true; + genPredOnRightSide = true; + break; + case LEFT: + genPredOnLeftSide = false; + genPredOnRightSide = true; + break; + case RIGHT: + genPredOnLeftSide = true; + genPredOnRightSide = false; + break; + default: return; } @@ -90,45 +105,58 @@ public void onMatch(RelOptRuleCall call) { } catch (CalciteSemanticException e) { return; } - + List leftJoinExprsList = new ArrayList<>(); List rightJoinExprsList = new ArrayList<>(); for (JoinLeafPredicateInfo joinLeafPredicateInfo : joinPredInfo.getEquiJoinPredicateElements()) { - leftJoinExprsList.addAll(joinLeafPredicateInfo.getJoinExprs(0)); - rightJoinExprsList.addAll(joinLeafPredicateInfo.getJoinExprs(1)); + leftJoinExprsList.addAll(joinLeafPredicateInfo.getJoinExprs(0)); + rightJoinExprsList.addAll(joinLeafPredicateInfo.getJoinExprs(1)); } // Build not null conditions final RelOptCluster cluster = join.getCluster(); final RexBuilder rexBuilder = join.getCluster().getRexBuilder(); + + RexNode newLeftPredicate = rexBuilder.makeLiteral(true); + RexNode newRightPredicate = rexBuilder.makeLiteral(true); + + // generate predicate on left side Set leftPushedPredicates = Sets.newHashSet(registry.getPushedPredicates(join, 0)); - final List newLeftConditions = getNotNullConditions(cluster, - rexBuilder, leftJoinExprsList, leftPushedPredicates); + if (genPredOnLeftSide) { + List newLeftConditions = getNotNullConditions(cluster, + rexBuilder, leftJoinExprsList, leftPushedPredicates); + newLeftPredicate = RexUtil.composeConjunction(rexBuilder, newLeftConditions, false); + } + + // generate predicate on right side Set rightPushedPredicates = Sets.newHashSet(registry.getPushedPredicates(join, 1)); - final List newRightConditions = getNotNullConditions(cluster, - rexBuilder, rightJoinExprsList, rightPushedPredicates); + if (genPredOnRightSide) { + List newRightConditions = getNotNullConditions(cluster, + rexBuilder, rightJoinExprsList, rightPushedPredicates); + newRightPredicate = RexUtil.composeConjunction(rexBuilder, newRightConditions, false); + } - // Nothing will be added to the expression - RexNode newLeftPredicate = RexUtil.composeConjunction(rexBuilder, newLeftConditions, false); - RexNode newRightPredicate = RexUtil.composeConjunction(rexBuilder, newRightConditions, false); - if (newLeftPredicate.isAlwaysTrue() && newRightPredicate.isAlwaysTrue()) { + boolean isNewLeftPredicateAlwaysTrue = newLeftPredicate.isAlwaysTrue(); + boolean isNewRightPredAlwaysTrue = newRightPredicate.isAlwaysTrue(); + + if (isNewLeftPredicateAlwaysTrue && isNewRightPredAlwaysTrue) { return; } - if (!newLeftPredicate.isAlwaysTrue()) { + if (!isNewLeftPredicateAlwaysTrue) { RelNode curr = lChild; lChild = filterFactory.createFilter(lChild, newLeftPredicate); call.getPlanner().onCopy(curr, lChild); } - if (!newRightPredicate.isAlwaysTrue()) { + if (!isNewRightPredAlwaysTrue) { RelNode curr = rChild; rChild = filterFactory.createFilter(rChild, newRightPredicate); call.getPlanner().onCopy(curr, rChild); } Join newJoin = join.copy(join.getTraitSet(), join.getCondition(), - lChild, rChild, join.getJoinType(), join.isSemiJoinDone()); + lChild, rChild, join.getJoinType(), join.isSemiJoinDone()); call.getPlanner().onCopy(join, newJoin); // Register information about created predicates @@ -139,18 +167,17 @@ public void onMatch(RelOptRuleCall call) { } private static List getNotNullConditions(RelOptCluster cluster, - RexBuilder rexBuilder, List inputJoinExprs, - Set pushedPredicates) { + RexBuilder rexBuilder, List inputJoinExprs, + Set pushedPredicates) { final List newConditions = Lists.newArrayList(); for (RexNode rexNode : inputJoinExprs) { - RexNode cond = rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, rexNode); - String digest = cond.toString(); - if (pushedPredicates.add(digest)) { - newConditions.add(cond); - } + RexNode cond = rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, rexNode); + String digest = cond.toString(); + if (pushedPredicates.add(digest)) { + newConditions.add(cond); + } } return newConditions; } - } diff --git a/ql/src/test/queries/clientpositive/transitive_not_null.q b/ql/src/test/queries/clientpositive/transitive_not_null.q new file mode 100644 index 0000000000..e5fb710e23 --- /dev/null +++ b/ql/src/test/queries/clientpositive/transitive_not_null.q @@ -0,0 +1,53 @@ +create table tbl_1(i1 int, j1 int); +insert into tbl_1 values(1,2),(1,null), (null, 200), (45,68); +create table tbl_2(i2 int, j2 int); +insert into tbl_2 values(1,2),(1,null), (null, 200), (45,68); + +-- simple join +explain cbo select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2; +select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2; + +explain cbo select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2; +select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2; + +explain cbo select * from tbl_1 full outer join tbl_2 on tbl_1.i1 = tbl_2.i2; +select * from tbl_1 full outer join tbl_2 on tbl_1.i1 = tbl_2.i2; + +-- conjunction +explain cbo select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2; +select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2; + +explain cbo select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2; +select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2; + +-- equi + non-equi +explain cbo select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2; +select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2; + +explain cbo select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2; +select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2; + +explain cbo SELECT t0.col0, t0.col1 +FROM + ( + SELECT i1 as col0, j1 as col1 FROM tbl_1 + ) AS t0 + LEFT JOIN + ( + SELECT i2 as col0, j2 as col1 FROM tbl_2 + ) AS t1 +ON t0.col0 = t1.col0 AND t0.col1 = t1.col1; + +SELECT t0.col0, t0.col1 +FROM + ( + SELECT i1 as col0, j1 as col1 FROM tbl_1 + ) AS t0 + LEFT JOIN + ( + SELECT i2 as col0, j2 as col1 FROM tbl_2 + ) AS t1 +ON t0.col0 = t1.col0 AND t0.col1 = t1.col1; + +DROP TABLE tbl_1; +DROP TABLE tbl_2; diff --git a/ql/src/test/results/clientpositive/annotate_stats_join.q.out b/ql/src/test/results/clientpositive/annotate_stats_join.q.out index 6c73d6398c..abe31e02a5 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_join.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_join.q.out @@ -814,16 +814,20 @@ STAGE PLANS: value expressions: _col2 (type: int) TableScan alias: dept_n1 + filterExpr: (deptid is not null and deptname is not null) (type: boolean) Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptid (type: int), deptname (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (deptid is not null and deptname is not null) (type: boolean) Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: int) + Select Operator + expressions: deptid (type: int), deptname (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) + Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE Reduce Operator Tree: Join Operator condition map: @@ -947,17 +951,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: emp_n2 + filterExpr: (deptid is not null and lastname is not null) (type: boolean) Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: lastname (type: string), deptid (type: int), locid (type: int) - outputColumnNames: _col0, _col1, _col2 + Filter Operator + predicate: (deptid is not null and lastname is not null) (type: boolean) Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Select Operator + expressions: lastname (type: string), deptid (type: int), locid (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int) + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: int) TableScan alias: dept_n1 Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/cbo_SortUnionTransposeRule.q.out b/ql/src/test/results/clientpositive/cbo_SortUnionTransposeRule.q.out index 1beaa648de..89beaa5d2a 100644 --- a/ql/src/test/results/clientpositive/cbo_SortUnionTransposeRule.q.out +++ b/ql/src/test/results/clientpositive/cbo_SortUnionTransposeRule.q.out @@ -296,17 +296,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src2 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) TableScan alias: src1 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE @@ -402,17 +406,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src2 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) TableScan alias: src1 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE @@ -911,17 +919,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src2 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: string) @@ -1045,17 +1057,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src2 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) TableScan Reduce Output Operator key expressions: _col0 (type: string) diff --git a/ql/src/test/results/clientpositive/cbo_rp_join0.q.out b/ql/src/test/results/clientpositive/cbo_rp_join0.q.out index 76eaa52855..8b072a5d25 100644 --- a/ql/src/test/results/clientpositive/cbo_rp_join0.q.out +++ b/ql/src/test/results/clientpositive/cbo_rp_join0.q.out @@ -735,17 +735,21 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 1615 Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: cbo_t4:cbo_t1 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), c_int (type: int) - outputColumnNames: key, c_int - Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: c_int (type: int) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), c_int (type: int) + outputColumnNames: key, c_int + Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: c_int (type: int) Reduce Operator Tree: Join Operator condition map: @@ -758,14 +762,14 @@ STAGE PLANS: 2 key (type: string) 3 key (type: string) outputColumnNames: key, c_int, key0, c_int0, key1, c_int2 - Statistics: Num rows: 1080 Data size: 288093 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 972 Data size: 259257 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), c_int (type: int), key0 (type: string), c_int0 (type: int), key1 (type: string), c_int2 (type: int) outputColumnNames: key, c_int, p, q, x, b - Statistics: Num rows: 1080 Data size: 288093 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 972 Data size: 259257 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1080 Data size: 288093 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 972 Data size: 259257 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/constant_prop_3.q.out b/ql/src/test/results/clientpositive/constant_prop_3.q.out index c95a56a55b..293d8f8a9d 100644 --- a/ql/src/test/results/clientpositive/constant_prop_3.q.out +++ b/ql/src/test/results/clientpositive/constant_prop_3.q.out @@ -355,10 +355,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: supplier_hive - filterExpr: (s_comment like '%Customer%Complaints%') (type: boolean) + filterExpr: ((s_comment like '%Customer%Complaints%') and s_suppkey is not null) (type: boolean) Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (s_comment like '%Customer%Complaints%') (type: boolean) + predicate: ((s_comment like '%Customer%Complaints%') and s_suppkey is not null) (type: boolean) Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: s_suppkey (type: int) diff --git a/ql/src/test/results/clientpositive/correlationoptimizer8.q.out b/ql/src/test/results/clientpositive/correlationoptimizer8.q.out index 69a6670f44..37a6c44be2 100644 --- a/ql/src/test/results/clientpositive/correlationoptimizer8.q.out +++ b/ql/src/test/results/clientpositive/correlationoptimizer8.q.out @@ -510,17 +510,21 @@ STAGE PLANS: value expressions: _col1 (type: bigint) TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -719,17 +723,21 @@ STAGE PLANS: value expressions: _col1 (type: bigint) TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Reduce Operator Tree: Demux Operator Statistics: Num rows: 112 Data size: 12648 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/infer_join_preds.q.out b/ql/src/test/results/clientpositive/infer_join_preds.q.out index 185d0f89de..11dbfa1a67 100644 --- a/ql/src/test/results/clientpositive/infer_join_preds.q.out +++ b/ql/src/test/results/clientpositive/infer_join_preds.q.out @@ -263,17 +263,21 @@ STAGE PLANS: value expressions: _col1 (type: string) TableScan alias: src1 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -579,17 +583,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) TableScan alias: src1 Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/innerjoin.q.out b/ql/src/test/results/clientpositive/innerjoin.q.out index 709dbdb539..9694ed1a72 100644 --- a/ql/src/test/results/clientpositive/innerjoin.q.out +++ b/ql/src/test/results/clientpositive/innerjoin.q.out @@ -1253,17 +1253,21 @@ STAGE PLANS: value expressions: _col1 (type: string) TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: diff --git a/ql/src/test/results/clientpositive/join45.q.out b/ql/src/test/results/clientpositive/join45.q.out index fd639b9d51..826412da1c 100644 --- a/ql/src/test/results/clientpositive/join45.q.out +++ b/ql/src/test/results/clientpositive/join45.q.out @@ -766,7 +766,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -811,17 +811,21 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: double) TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -890,7 +894,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src JOIN src1 a ON (a.key+src.key >= 100) @@ -919,7 +923,7 @@ POSTHOOK: Input: default@src1 98 val_98 273 val_273 98 val_98 98 val_98 150 val_150 98 val_98 98 val_98 401 val_401 98 val_98 -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -964,17 +968,21 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: double) TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1043,7 +1051,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src LEFT OUTER JOIN src1 a ON (a.key = src.key) @@ -1072,7 +1080,7 @@ POSTHOOK: Input: default@src1 35 val_35 NULL NULL 66 val_66 35 val_35 NULL NULL 66 val_66 35 val_35 NULL NULL 66 val_66 -Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -1104,15 +1112,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) TableScan alias: a Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE @@ -1196,7 +1208,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src JOIN src1 a ON (a.key+src.key >= 100) @@ -1225,7 +1237,7 @@ NULL NULL NULL NULL val_409 NULL NULL NULL NULL NULL NULL NULL NULL val_193 NULL NULL NULL NULL val_265 -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -1257,17 +1269,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: double) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: double) TableScan alias: a Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE @@ -1349,7 +1365,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src RIGHT OUTER JOIN src1 a ON (a.key = src.key) @@ -1674,9 +1690,9 @@ POSTHOOK: Input: default@src1 35 val_35 NULL NULL 66 val_66 35 val_35 NULL NULL 66 val_66 35 val_35 NULL NULL 66 val_66 -Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product -Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM cbo_t1 t1 @@ -1796,7 +1812,7 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 100 Data size: 72921 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 90 Data size: 65601 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: int), _col9 (type: float), _col10 (type: boolean), _col11 (type: string) Reduce Operator Tree: Join Operator @@ -1806,14 +1822,14 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 - Statistics: Num rows: 800000 Data size: 1467199175 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 720000 Data size: 1320479175 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (((_col0 = _col25) or ((_col29 = _col17) and _col6)) and ((_col13 = _col25) or ((_col28 = _col16) and (_col14 = _col26))) and ((_col7 = _col25) or ((_col8 = _col26) and (_col27 = _col21)))) (type: boolean) - Statistics: Num rows: 337500 Data size: 618975000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 303750 Data size: 557077500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col19 (type: string), _col20 (type: string), _col21 (type: int), _col22 (type: float), _col23 (type: boolean), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: float), _col29 (type: boolean), _col30 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: int), _col10 (type: float), _col11 (type: boolean), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: int), _col16 (type: float), _col17 (type: boolean), _col18 (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29 - Statistics: Num rows: 337500 Data size: 617625000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 303750 Data size: 555862500 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 18300 Basic stats: COMPLETE Column stats: COMPLETE @@ -1830,17 +1846,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 20 Data size: 7138 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), c_int (type: int), c_float (type: float), c_boolean (type: boolean), dt (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 20 Data size: 7138 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 7138 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 18 Data size: 6406 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), c_int (type: int), c_float (type: float), c_boolean (type: boolean), dt (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 18 Data size: 6406 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 18 Data size: 6406 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string) TableScan alias: t2 Statistics: Num rows: 20 Data size: 7138 Basic stats: COMPLETE Column stats: COMPLETE @@ -1862,7 +1882,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 100 Data size: 72921 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 90 Data size: 65601 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false table: @@ -1876,9 +1896,9 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product -Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM cbo_t1 t1 RIGHT OUTER JOIN cbo_t1 t2 ON (t2.key = t1.key) @@ -1902,8 +1922,8 @@ POSTHOOK: Input: default@cbo_t1 POSTHOOK: Input: default@cbo_t1@dt=2014 #### A masked pattern was here #### 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 -1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 +1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 diff --git a/ql/src/test/results/clientpositive/join46.q.out b/ql/src/test/results/clientpositive/join46.q.out index 02cb625d0f..3649968328 100644 --- a/ql/src/test/results/clientpositive/join46.q.out +++ b/ql/src/test/results/clientpositive/join46.q.out @@ -79,17 +79,21 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string) TableScan alias: test2_n0 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -98,10 +102,10 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -178,10 +182,10 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: boolean) TableScan alias: test2_n0 - filterExpr: key BETWEEN 100 AND 102 (type: boolean) + filterExpr: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) + predicate: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), value (type: int), col_2 (type: string) @@ -379,17 +383,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test1_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) TableScan alias: test2_n0 Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE @@ -411,10 +419,10 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -909,17 +917,21 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: boolean) TableScan alias: test2_n0 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -929,14 +941,14 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1420,17 +1432,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test1_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) TableScan alias: test2_n0 Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE @@ -1453,14 +1469,14 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1910,7 +1926,7 @@ NULL NULL NULL 105 NULL None 101 2 Car 102 2 Del 99 2 Mat 102 2 Del NULL NULL NULL 104 3 Fli -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM ( @@ -1967,19 +1983,6 @@ STAGE PLANS: Stage: Stage-1 Map Reduce Map Operator Tree: - TableScan - alias: test1_n2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) TableScan alias: test2_n0 Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE @@ -1993,20 +1996,37 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: boolean) + TableScan + alias: test1_n2 + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Reduce Operator Tree: Join Operator condition map: - Right Outer Join 0 to 1 + Left Outer Join 0 to 1 keys: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + residual filter predicates: {(_col4 BETWEEN 100 AND 102 or _col3)} + Statistics: Num rows: 8 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + expressions: _col4 (type: int), _col5 (type: int), _col6 (type: string), _col0 (type: int), _col1 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false table: @@ -2020,12 +2040,12 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) TableScan Reduce Output Operator sort order: - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Reduce Operator Tree: Join Operator @@ -2036,10 +2056,10 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 72 Data size: 27306 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 72 Data size: 27306 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2063,17 +2083,21 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: boolean) TableScan alias: test2_n0 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -2083,11 +2107,11 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false table: @@ -2101,7 +2125,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1_n2.key AS key1, test1_n2.value AS value1, test1_n2.col_1 AS col_1, diff --git a/ql/src/test/results/clientpositive/join47.q.out b/ql/src/test/results/clientpositive/join47.q.out index e9b6be4f3a..02b4e7e5ae 100644 --- a/ql/src/test/results/clientpositive/join47.q.out +++ b/ql/src/test/results/clientpositive/join47.q.out @@ -758,7 +758,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -803,17 +803,21 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: double) TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -880,7 +884,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src JOIN src1 a ON (a.key+src.key >= 100) @@ -909,7 +913,7 @@ POSTHOOK: Input: default@src1 98 val_98 273 val_273 98 val_98 98 val_98 150 val_150 98 val_98 98 val_98 401 val_401 98 val_98 -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -954,17 +958,21 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: double) TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1031,7 +1039,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src LEFT OUTER JOIN src1 a ON (a.key = src.key) @@ -1060,7 +1068,7 @@ POSTHOOK: Input: default@src1 35 val_35 NULL NULL 66 val_66 35 val_35 NULL NULL 66 val_66 35 val_35 NULL NULL 66 val_66 -Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -1092,15 +1100,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) TableScan alias: a Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE @@ -1182,7 +1194,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src JOIN src1 a ON (a.key+src.key >= 100) @@ -1211,7 +1223,7 @@ NULL NULL NULL NULL val_409 NULL NULL NULL NULL NULL NULL NULL NULL val_193 NULL NULL NULL NULL val_265 -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -1243,17 +1255,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: double) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: double) TableScan alias: a Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE @@ -1333,7 +1349,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src RIGHT OUTER JOIN src1 a ON (a.key = src.key) @@ -1654,9 +1670,9 @@ POSTHOOK: Input: default@src1 35 val_35 NULL NULL 66 val_66 35 val_35 NULL NULL 66 val_66 35 val_35 NULL NULL 66 val_66 -Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product -Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM cbo_t1 t1 @@ -1776,7 +1792,7 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 100 Data size: 72921 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 90 Data size: 65601 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: int), _col9 (type: float), _col10 (type: boolean), _col11 (type: string) Reduce Operator Tree: Join Operator @@ -1787,11 +1803,11 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 residual filter predicates: {((_col0 = _col25) or ((_col29 = _col17) and _col6))} {((_col13 = _col25) or ((_col28 = _col16) and (_col14 = _col26)))} {((_col7 = _col25) or ((_col8 = _col26) and (_col27 = _col21)))} - Statistics: Num rows: 337500 Data size: 618975000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 303750 Data size: 557077500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col19 (type: string), _col20 (type: string), _col21 (type: int), _col22 (type: float), _col23 (type: boolean), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: float), _col29 (type: boolean), _col30 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: int), _col10 (type: float), _col11 (type: boolean), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: int), _col16 (type: float), _col17 (type: boolean), _col18 (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29 - Statistics: Num rows: 337500 Data size: 617625000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 303750 Data size: 555862500 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 18300 Basic stats: COMPLETE Column stats: COMPLETE @@ -1808,17 +1824,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 20 Data size: 7138 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), c_int (type: int), c_float (type: float), c_boolean (type: boolean), dt (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 20 Data size: 7138 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 7138 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 18 Data size: 6406 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), c_int (type: int), c_float (type: float), c_boolean (type: boolean), dt (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 18 Data size: 6406 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 18 Data size: 6406 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string) TableScan alias: t2 Statistics: Num rows: 20 Data size: 7138 Basic stats: COMPLETE Column stats: COMPLETE @@ -1840,7 +1860,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 100 Data size: 72921 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 90 Data size: 65601 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false table: @@ -1854,9 +1874,9 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product -Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM cbo_t1 t1 RIGHT OUTER JOIN cbo_t1 t2 ON (t2.key = t1.key) @@ -1880,8 +1900,8 @@ POSTHOOK: Input: default@cbo_t1 POSTHOOK: Input: default@cbo_t1@dt=2014 #### A masked pattern was here #### 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 -1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 +1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 false 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 diff --git a/ql/src/test/results/clientpositive/join_cond_pushdown_unqual5.q.out b/ql/src/test/results/clientpositive/join_cond_pushdown_unqual5.q.out index a966d8caf3..ea2535775b 100644 --- a/ql/src/test/results/clientpositive/join_cond_pushdown_unqual5.q.out +++ b/ql/src/test/results/clientpositive/join_cond_pushdown_unqual5.q.out @@ -70,17 +70,21 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int) TableScan alias: r + filterExpr: (rk1 is not null and rk2 is not null) (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ra (type: int), rk1 (type: string), rk2 (type: string) - outputColumnNames: _col0, _col1, _col2 + Filter Operator + predicate: (rk1 is not null and rk2 is not null) (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col2 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col2 (type: string) + Select Operator + expressions: ra (type: int), rk1 (type: string), rk2 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) + Reduce Output Operator + key expressions: _col1 (type: string), _col2 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) Reduce Operator Tree: Join Operator condition map: @@ -144,15 +148,19 @@ STAGE PLANS: $hdt$_1:r TableScan alias: r + filterExpr: (rk1 is not null and rk2 is not null) (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ra (type: int), rk1 (type: string), rk2 (type: string) - outputColumnNames: _col0, _col1, _col2 + Filter Operator + predicate: (rk1 is not null and rk2 is not null) (type: boolean) Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col2 (type: string), _col3 (type: string) - 1 _col1 (type: string), _col2 (type: string) + Select Operator + expressions: ra (type: int), rk1 (type: string), rk2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col2 (type: string), _col3 (type: string) + 1 _col1 (type: string), _col2 (type: string) Stage: Stage-3 Map Reduce diff --git a/ql/src/test/results/clientpositive/join_emit_interval.q.out b/ql/src/test/results/clientpositive/join_emit_interval.q.out index 9f3f01f57e..ef406607b1 100644 --- a/ql/src/test/results/clientpositive/join_emit_interval.q.out +++ b/ql/src/test/results/clientpositive/join_emit_interval.q.out @@ -79,17 +79,21 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: boolean) TableScan alias: test2_n4 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -101,14 +105,14 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/join_filters_overlap.q.out b/ql/src/test/results/clientpositive/join_filters_overlap.q.out index 41ebfccbbc..4eab694c2a 100644 --- a/ql/src/test/results/clientpositive/join_filters_overlap.q.out +++ b/ql/src/test/results/clientpositive/join_filters_overlap.q.out @@ -23,10 +23,10 @@ FROM (SELECT `key`, `value`, `value` = 60 AS `=`, `value` = 50 AS `=3` FROM `default`.`a_n4`) AS `t` LEFT JOIN (SELECT `key`, CAST(50 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 50) AS `t1` ON `t`.`key` = `t1`.`key` AND `t`.`=3` +WHERE `value` = 50 AND `key` IS NOT NULL) AS `t1` ON `t`.`key` = `t1`.`key` AND `t`.`=3` LEFT JOIN (SELECT `key`, CAST(60 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 60) AS `t3` ON `t`.`key` = `t3`.`key` AND `t`.`=` +WHERE `value` = 60 AND `key` IS NOT NULL) AS `t3` ON `t`.`key` = `t3`.`key` AND `t`.`=` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -55,12 +55,12 @@ STAGE PLANS: auto parallelism: false TableScan alias: b - filterExpr: (value = 50) (type: boolean) + filterExpr: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 50) (type: boolean) + predicate: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), 50 (type: int) @@ -178,12 +178,12 @@ STAGE PLANS: auto parallelism: false TableScan alias: c - filterExpr: (value = 60) (type: boolean) + filterExpr: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 60) (type: boolean) + predicate: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), 60 (type: int) @@ -355,12 +355,12 @@ POSTHOOK: Input: default@a_n4 OPTIMIZED SQL: SELECT `t0`.`key`, `t0`.`value`, `t1`.`key` AS `key1`, `t1`.`value` AS `value1`, `t3`.`key` AS `key2`, `t3`.`value` AS `value2` FROM (SELECT `key`, CAST(50 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 50) AS `t0` +WHERE `value` = 50 AND `key` IS NOT NULL) AS `t0` RIGHT JOIN (SELECT `key`, `value`, `value` = 60 AS `=`, `value` = 50 AS `=3` FROM `default`.`a_n4`) AS `t1` ON `t0`.`key` = `t1`.`key` AND `t1`.`=3` LEFT JOIN (SELECT `key`, CAST(60 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 60) AS `t3` ON `t1`.`key` = `t3`.`key` AND `t1`.`=` +WHERE `value` = 60 AND `key` IS NOT NULL) AS `t3` ON `t1`.`key` = `t3`.`key` AND `t1`.`=` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -372,12 +372,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a_n4 - filterExpr: (value = 50) (type: boolean) + filterExpr: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 50) (type: boolean) + predicate: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), 50 (type: int) @@ -512,12 +512,12 @@ STAGE PLANS: auto parallelism: false TableScan alias: c - filterExpr: (value = 60) (type: boolean) + filterExpr: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 60) (type: boolean) + predicate: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), 60 (type: int) @@ -689,12 +689,12 @@ POSTHOOK: Input: default@a_n4 OPTIMIZED SQL: SELECT `t0`.`key`, `t0`.`value`, `t1`.`key` AS `key1`, `t1`.`value` AS `value1`, `t3`.`key` AS `key2`, `t3`.`value` AS `value2` FROM (SELECT `key`, CAST(50 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 50) AS `t0` +WHERE `value` = 50 AND `key` IS NOT NULL) AS `t0` RIGHT JOIN (SELECT `key`, `value`, `value` = 60 AS `=`, `value` = 50 AS `=3` FROM `default`.`a_n4`) AS `t1` ON `t0`.`key` = `t1`.`key` AND `t1`.`=3` LEFT JOIN (SELECT `key`, CAST(60 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 60) AS `t3` ON `t1`.`key` = `t3`.`key` AND `t1`.`=` +WHERE `value` = 60 AND `key` IS NOT NULL) AS `t3` ON `t1`.`key` = `t3`.`key` AND `t1`.`=` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -706,12 +706,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a_n4 - filterExpr: (value = 50) (type: boolean) + filterExpr: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 50) (type: boolean) + predicate: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), 50 (type: int) @@ -846,12 +846,12 @@ STAGE PLANS: auto parallelism: false TableScan alias: c - filterExpr: (value = 60) (type: boolean) + filterExpr: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 60) (type: boolean) + predicate: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), 60 (type: int) @@ -1027,10 +1027,10 @@ FULL JOIN (SELECT `key`, `value`, `value` = 60 AS `=`, `value` = 50 AS `=3` FROM `default`.`a_n4`) AS `t0` ON `t`.`key` = `t0`.`key` AND `t`.`=3` AND `t0`.`=3` LEFT JOIN (SELECT `key`, CAST(60 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 60) AS `t2` ON `t0`.`key` = `t2`.`key` AND `t0`.`=` +WHERE `value` = 60 AND `key` IS NOT NULL) AS `t2` ON `t0`.`key` = `t2`.`key` AND `t0`.`=` LEFT JOIN (SELECT `key`, CAST(40 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 40) AS `t4` ON `t`.`key` = `t4`.`key` AND `t`.`=` +WHERE `value` = 40 AND `key` IS NOT NULL) AS `t4` ON `t`.`key` = `t4`.`key` AND `t`.`=` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -1179,12 +1179,12 @@ STAGE PLANS: auto parallelism: false TableScan alias: c - filterExpr: (value = 60) (type: boolean) + filterExpr: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 60) (type: boolean) + predicate: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), 60 (type: int) @@ -1325,12 +1325,12 @@ STAGE PLANS: auto parallelism: false TableScan alias: d - filterExpr: (value = 40) (type: boolean) + filterExpr: ((value = 40) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 40) (type: boolean) + predicate: ((value = 40) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), 40 (type: int) @@ -1495,13 +1495,13 @@ FROM (SELECT `key`, `value`, `value` = 40 AS `=`, `value` = 60 AS `=3`, `value` FROM `default`.`a_n4`) AS `t` LEFT JOIN (SELECT `key`, CAST(50 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 50) AS `t1` ON `t`.`key` = `t1`.`key` AND `t`.`=4` +WHERE `value` = 50 AND `key` IS NOT NULL) AS `t1` ON `t`.`key` = `t1`.`key` AND `t`.`=4` LEFT JOIN (SELECT `key`, CAST(60 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 60) AS `t3` ON `t`.`key` = `t3`.`key` AND `t`.`=3` +WHERE `value` = 60 AND `key` IS NOT NULL) AS `t3` ON `t`.`key` = `t3`.`key` AND `t`.`=3` LEFT JOIN (SELECT `key`, CAST(40 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 40) AS `t5` ON `t`.`key` = `t5`.`key` AND `t`.`=` +WHERE `value` = 40 AND `key` IS NOT NULL) AS `t5` ON `t`.`key` = `t5`.`key` AND `t`.`=` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -1531,12 +1531,12 @@ STAGE PLANS: auto parallelism: false TableScan alias: b - filterExpr: (value = 50) (type: boolean) + filterExpr: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 50) (type: boolean) + predicate: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), 50 (type: int) @@ -1654,12 +1654,12 @@ STAGE PLANS: auto parallelism: false TableScan alias: c - filterExpr: (value = 60) (type: boolean) + filterExpr: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 60) (type: boolean) + predicate: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), 60 (type: int) @@ -1800,12 +1800,12 @@ STAGE PLANS: auto parallelism: false TableScan alias: d - filterExpr: (value = 40) (type: boolean) + filterExpr: ((value = 40) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 40) (type: boolean) + predicate: ((value = 40) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), 40 (type: int) diff --git a/ql/src/test/results/clientpositive/join_merging.q.out b/ql/src/test/results/clientpositive/join_merging.q.out index 5b9c0630e6..2688f8389e 100644 --- a/ql/src/test/results/clientpositive/join_merging.q.out +++ b/ql/src/test/results/clientpositive/join_merging.q.out @@ -23,23 +23,27 @@ STAGE PLANS: Map Operator Tree: TableScan alias: p2 + filterExpr: p_partkey is not null (type: boolean) Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: p_partkey is not null (type: boolean) Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) TableScan alias: p1 - filterExpr: (p_size > 10) (type: boolean) + filterExpr: ((p_size > 10) and p_partkey is not null) (type: boolean) Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (p_size > 10) (type: boolean) + predicate: ((p_size > 10) and p_partkey is not null) (type: boolean) Statistics: Num rows: 8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_partkey (type: int), p_size (type: int) @@ -54,16 +58,16 @@ STAGE PLANS: Reduce Operator Tree: Join Operator condition map: - Right Outer Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 16 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: int), _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 16 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false table: @@ -91,7 +95,7 @@ STAGE PLANS: key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 16 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: int) Reduce Operator Tree: Join Operator @@ -101,14 +105,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col1 (type: int) outputColumnNames: _col1, _col3 - Statistics: Num rows: 42 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 34 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int), _col3 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 42 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 34 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 42 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 34 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/join_star.q.out b/ql/src/test/results/clientpositive/join_star.q.out index 9caf12da49..5702ff0be4 100644 --- a/ql/src/test/results/clientpositive/join_star.q.out +++ b/ql/src/test/results/clientpositive/join_star.q.out @@ -528,27 +528,35 @@ STAGE PLANS: $hdt$_1:dim1 TableScan alias: dim1 + filterExpr: f1 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f1 (type: int), f2 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f1 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col2 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f1 (type: int), f2 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col2 (type: int) + 1 _col0 (type: int) $hdt$_2:dim2 TableScan alias: dim2 + filterExpr: f3 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f3 (type: int), f4 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f3 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col4 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f3 (type: int), f4 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col4 (type: int) + 1 _col0 (type: int) Stage: Stage-5 Map Reduce @@ -691,87 +699,115 @@ STAGE PLANS: $hdt$_1:dim1 TableScan alias: dim1 + filterExpr: f1 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f1 (type: int), f2 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f1 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col2 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f1 (type: int), f2 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col2 (type: int) + 1 _col0 (type: int) $hdt$_2:dim2 TableScan alias: dim2 + filterExpr: f3 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f3 (type: int), f4 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f3 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col5 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f3 (type: int), f4 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col5 (type: int) + 1 _col0 (type: int) $hdt$_3:dim3 TableScan alias: dim3 + filterExpr: f5 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f5 (type: int), f6 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f5 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col3 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f5 (type: int), f6 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col3 (type: int) + 1 _col0 (type: int) $hdt$_4:dim4 TableScan alias: dim4 + filterExpr: f7 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f7 (type: int), f8 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f7 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col9 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f7 (type: int), f8 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col9 (type: int) + 1 _col0 (type: int) $hdt$_5:dim5 TableScan alias: dim5 + filterExpr: f9 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f9 (type: int), f10 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f9 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col11 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f9 (type: int), f10 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col11 (type: int) + 1 _col0 (type: int) $hdt$_6:dim6 TableScan alias: dim6 + filterExpr: f11 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f11 (type: int), f12 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f11 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col9 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f11 (type: int), f12 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col9 (type: int) + 1 _col0 (type: int) $hdt$_7:dim7 TableScan alias: dim7 + filterExpr: f13 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f13 (type: int), f14 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f13 is not null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col15 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f13 (type: int), f14 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col15 (type: int) + 1 _col0 (type: int) Stage: Stage-15 Map Reduce diff --git a/ql/src/test/results/clientpositive/lineage1.q.out b/ql/src/test/results/clientpositive/lineage1.q.out index 4a2ca453ac..e63bb24a49 100644 --- a/ql/src/test/results/clientpositive/lineage1.q.out +++ b/ql/src/test/results/clientpositive/lineage1.q.out @@ -56,17 +56,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: p1 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) TableScan alias: t1 Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE @@ -236,17 +240,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: p2 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) TableScan alias: t2 Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out index cf7252f753..0799a41aa8 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_14.q.out @@ -62,11 +62,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Map Operator Tree: TableScan alias: a @@ -164,11 +168,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE Map Operator Tree: TableScan alias: b diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out index b6e0ebf30a..c2809d3025 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_15.q.out @@ -62,11 +62,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Map Operator Tree: TableScan alias: a @@ -143,11 +147,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE Map Operator Tree: TableScan alias: b diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out index 4e156926d6..68428156f1 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out @@ -222,18 +222,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: bucket_small_n17 - filterExpr: pri BETWEEN 1L AND 2L (type: boolean) + filterExpr: key is not null (type: boolean) Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: bigint), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 225 Data size: 43508 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: bigint), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 225 Data size: 43508 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 225 Data size: 43508 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -246,14 +249,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 247 Data size: 47858 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 247 Data size: 47858 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 247 Data size: 47858 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -370,12 +373,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: bucket_small_n17 - filterExpr: pri BETWEEN 1L AND 2L (type: boolean) + filterExpr: key is not null (type: boolean) Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: bigint), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 225 Data size: 43508 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: bigint), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 225 Data size: 43508 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: bucket_big_n17 @@ -392,14 +398,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 247 Data size: 47858 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 247 Data size: 47858 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 247 Data size: 47858 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -626,12 +632,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: bucket_small_n17 - filterExpr: pri BETWEEN 1L AND 2L (type: boolean) + filterExpr: key is not null (type: boolean) Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: bigint), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 225 Data size: 43508 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: bigint), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 225 Data size: 43508 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: bucket_big_n17 @@ -648,14 +657,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 247 Data size: 47858 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 247 Data size: 47858 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 247 Data size: 47858 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/check_constraint.q.out b/ql/src/test/results/clientpositive/llap/check_constraint.q.out index 7b794ba34b..297d8928f8 100644 --- a/ql/src/test/results/clientpositive/llap/check_constraint.q.out +++ b/ql/src/test/results/clientpositive/llap/check_constraint.q.out @@ -1563,17 +1563,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s2 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), CAST( key AS decimal(5,2)) (type: decimal(5,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 99500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), CAST( key AS decimal(5,2)) (type: decimal(5,2)) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 99500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: decimal(5,2)) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 99500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(5,2)) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out b/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out index eef365b9b1..5653d40f3a 100644 --- a/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out +++ b/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out @@ -57,37 +57,39 @@ Stage-0 limit:-1 Stage-1 Reducer 2 llap - File Output Operator [FS_17] - Merge Join Operator [MERGEJOIN_21] (rows=1 width=4) - Conds:RS_33._col0=RS_36._col0(Left Outer),Output:["_col0"] + File Output Operator [FS_19] + Merge Join Operator [MERGEJOIN_23] (rows=1 width=4) + Conds:RS_35._col0=RS_39._col0(Left Outer),Output:["_col0"] <-Map 1 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_33] + SHUFFLE [RS_35] PartitionCols:_col0 - Select Operator [SEL_32] (rows=1 width=4) + Select Operator [SEL_34] (rows=1 width=4) Output:["_col0"] TableScan [TS_0] (rows=1 width=4) default@tb1,a,Tbl:COMPLETE,Col:NONE,Output:["id"] <-Reducer 5 [SIMPLE_EDGE] vectorized, llap - SHUFFLE [RS_36] + SHUFFLE [RS_39] PartitionCols:_col0 - Limit [LIM_35] (rows=1 width=2) - Number of rows:1 - Select Operator [SEL_34] (rows=1 width=2) - Output:["_col0"] - <-Union 4 [CUSTOM_SIMPLE_EDGE] - <-Map 3 [CONTAINS] vectorized, llap - Reduce Output Operator [RS_39] - Limit [LIM_38] (rows=1 width=2) - Number of rows:1 - Select Operator [SEL_37] (rows=1 width=4) - Output:["_col0"] - TableScan [TS_22] (rows=1 width=4) - Output:["id"] - <-Map 6 [CONTAINS] vectorized, llap - Reduce Output Operator [RS_42] - Limit [LIM_41] (rows=1 width=2) - Number of rows:1 - Select Operator [SEL_40] (rows=1 width=0) - Output:["_col0"] - TableScan [TS_27] (rows=1 width=0) + Filter Operator [FIL_38] (rows=1 width=2) + predicate:_col0 is not null + Limit [LIM_37] (rows=1 width=2) + Number of rows:1 + Select Operator [SEL_36] (rows=1 width=2) + Output:["_col0"] + <-Union 4 [CUSTOM_SIMPLE_EDGE] + <-Map 3 [CONTAINS] vectorized, llap + Reduce Output Operator [RS_42] + Limit [LIM_41] (rows=1 width=2) + Number of rows:1 + Select Operator [SEL_40] (rows=1 width=4) + Output:["_col0"] + TableScan [TS_24] (rows=1 width=4) + Output:["id"] + <-Map 6 [CONTAINS] vectorized, llap + Reduce Output Operator [RS_45] + Limit [LIM_44] (rows=1 width=2) + Number of rows:1 + Select Operator [SEL_43] (rows=1 width=0) + Output:["_col0"] + TableScan [TS_29] (rows=1 width=0) diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out index 4ba041d992..f21e33a434 100644 --- a/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out +++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer1.q.out @@ -846,16 +846,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -996,16 +1000,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -1146,16 +1154,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -1296,16 +1308,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -1443,16 +1459,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -1581,16 +1601,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -1706,16 +1730,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 5 @@ -1856,16 +1884,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 5 @@ -2006,16 +2038,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 5 @@ -2156,16 +2192,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 5 diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out index c0909fe1ad..2f6282ea28 100644 --- a/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out +++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out @@ -379,20 +379,24 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -535,20 +539,24 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 12 Data size: 1128 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -667,20 +675,24 @@ STAGE PLANS: Map Operator Tree: TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs Map 3 @@ -823,20 +835,24 @@ STAGE PLANS: Map Operator Tree: TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs Map 3 diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out index 66e5bff966..45a646c948 100644 --- a/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out +++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer4.q.out @@ -790,16 +790,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -815,16 +819,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: z + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1027,16 +1035,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1052,16 +1064,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: z + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1264,16 +1280,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1289,16 +1309,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: z + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1500,16 +1524,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1525,16 +1553,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1736,16 +1768,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1761,16 +1797,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1972,16 +2012,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1997,16 +2041,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index 87450c55e6..6f1f20f174 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -3145,6 +3145,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) @@ -3234,6 +3235,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) @@ -5856,6 +5858,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) @@ -5925,6 +5928,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 7c45ac2e66..8d507909e5 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -1125,25 +1125,27 @@ Stage-0 limit:-1 Stage-1 Reducer 2 llap - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=85 width=7) + File Output Operator [FS_9] + Select Operator [SEL_8] (rows=77 width=7) Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_16] (rows=85 width=7) - Conds:RS_4._col0=RS_5._col0(Left Outer),Output:["_col1","_col3"] + Merge Join Operator [MERGEJOIN_17] (rows=77 width=7) + Conds:RS_5._col0=RS_6._col0(Left Outer),Output:["_col1","_col3"] <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_4] + SHUFFLE [RS_5] PartitionCols:_col0 Select Operator [SEL_1] (rows=20 width=84) Output:["_col0","_col1"] TableScan [TS_0] (rows=20 width=84) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] <-Map 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_5] + SHUFFLE [RS_6] PartitionCols:_col0 - Select Operator [SEL_3] (rows=20 width=84) + Select Operator [SEL_4] (rows=18 width=84) Output:["_col0","_col1"] - TableScan [TS_2] (rows=20 width=84) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Filter Operator [FIL_11] (rows=18 width=84) + predicate:key is not null + TableScan [TS_2] (rows=20 width=84) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] PREHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 full outer join cbo_t2 on cbo_t1.key=cbo_t2.key PREHOOK: type: QUERY @@ -2786,24 +2788,24 @@ Stage-0 limit:-1 Stage-1 Reducer 4 llap - File Output Operator [FS_32] - Select Operator [SEL_31] (rows=27 width=125) + File Output Operator [FS_34] + Select Operator [SEL_33] (rows=27 width=125) Output:["_col0","_col1"] <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] - Select Operator [SEL_29] (rows=27 width=125) + SHUFFLE [RS_32] + Select Operator [SEL_31] (rows=27 width=125) Output:["_col0","_col1"] - Filter Operator [FIL_28] (rows=27 width=141) + Filter Operator [FIL_30] (rows=27 width=141) predicate:(((_col3 >= _col2) or (_col2 = 0L) or _col5 is not null or _col1 is null) and (_col1 is not null or (_col2 = 0L) or _col5 is not null) and (_col5 is null or (_col2 = 0L))) - Select Operator [SEL_27] (rows=27 width=141) + Select Operator [SEL_29] (rows=27 width=141) Output:["_col0","_col1","_col2","_col3","_col5"] - Merge Join Operator [MERGEJOIN_38] (rows=27 width=141) + Merge Join Operator [MERGEJOIN_40] (rows=27 width=141) Conds:(Inner),Output:["_col0","_col1","_col3","_col4","_col5"] <-Reducer 6 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_25] - Group By Operator [GBY_19] (rows=1 width=16) + PARTITION_ONLY_SHUFFLE [RS_27] + Group By Operator [GBY_21] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["count()","count(_col0)"] - Select Operator [SEL_15] (rows=1 width=16) + Select Operator [SEL_17] (rows=1 width=16) Output:["_col0"] Group By Operator [GBY_7] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"] @@ -2811,22 +2813,24 @@ Stage-0 PARTITION_ONLY_SHUFFLE [RS_6] Group By Operator [GBY_5] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(p_size)","count(p_size)"] - Filter Operator [FIL_35] (rows=8 width=4) + Filter Operator [FIL_37] (rows=8 width=4) predicate:(p_size < 10) TableScan [TS_2] (rows=26 width=4) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_24] - Merge Join Operator [MERGEJOIN_37] (rows=27 width=125) - Conds:RS_21.UDFToDouble(_col1)=RS_22._col0(Left Outer),Output:["_col0","_col1","_col3"] + PARTITION_ONLY_SHUFFLE [RS_26] + Merge Join Operator [MERGEJOIN_39] (rows=27 width=125) + Conds:RS_23.UDFToDouble(_col1)=RS_24._col0(Left Outer),Output:["_col0","_col1","_col3"] <-Reducer 6 [SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_22] + PARTITION_ONLY_SHUFFLE [RS_24] PartitionCols:_col0 - Select Operator [SEL_8] (rows=1 width=12) + Select Operator [SEL_10] (rows=1 width=12) Output:["_col0","_col1"] - Please refer to the previous Group By Operator [GBY_7] + Filter Operator [FIL_9] (rows=1 width=16) + predicate:(UDFToDouble(_col0) is not null and _col1 is not null) + Please refer to the previous Group By Operator [GBY_7] <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] + SHUFFLE [RS_23] PartitionCols:UDFToDouble(_col1) Select Operator [SEL_1] (rows=26 width=125) Output:["_col0","_col1"] @@ -6497,27 +6501,29 @@ Stage-0 limit:-1 Stage-1 Reducer 3 llap - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=24) + File Output Operator [FS_15] + Group By Operator [GBY_13] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=24) + PARTITION_ONLY_SHUFFLE [RS_12] + Group By Operator [GBY_11] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_9] + PARTITION_ONLY_SHUFFLE [RS_10] PartitionCols:rand() - Select Operator [SEL_7] (rows=1 width=404) + Select Operator [SEL_8] (rows=1 width=404) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_22] (rows=1 width=404) - Conds:SEL_1._col2=RS_5._col1(Left Outer),Output:["_col0","_col1","_col3"] + Map Join Operator [MAPJOIN_23] (rows=1 width=404) + Conds:SEL_1._col2=RS_6._col1(Left Outer),Output:["_col0","_col1","_col3"] <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_5] + BROADCAST [RS_6] PartitionCols:_col1 - Select Operator [SEL_3] (rows=1 width=184) + Select Operator [SEL_4] (rows=1 width=184) Output:["_col0","_col1"] - TableScan [TS_2] (rows=1 width=184) - default@t2_n70,c,Tbl:COMPLETE,Col:NONE,Output:["key"] + Filter Operator [FIL_17] (rows=1 width=184) + predicate:UDFToDouble(key) is not null + TableScan [TS_2] (rows=1 width=184) + default@t2_n70,c,Tbl:COMPLETE,Col:NONE,Output:["key"] <-Select Operator [SEL_1] (rows=1 width=368) Output:["_col0","_col1","_col2"] TableScan [TS_0] (rows=1 width=368) @@ -6545,30 +6551,32 @@ Stage-0 limit:-1 Stage-1 Reducer 4 llap - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=24) + File Output Operator [FS_15] + Group By Operator [GBY_13] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=24) + PARTITION_ONLY_SHUFFLE [RS_12] + Group By Operator [GBY_11] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] <-Map 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_9] + PARTITION_ONLY_SHUFFLE [RS_10] PartitionCols:rand() - Select Operator [SEL_7] (rows=1 width=404) + Select Operator [SEL_8] (rows=1 width=404) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_17] (rows=1 width=404) - Conds:RS_4._col2=SEL_3._col1(Right Outer),Output:["_col0","_col1","_col3"] + Map Join Operator [MAPJOIN_18] (rows=1 width=404) + Conds:RS_5._col2=SEL_4._col1(Right Outer),Output:["_col0","_col1","_col3"] <-Map 1 [BROADCAST_EDGE] llap - BROADCAST [RS_4] + BROADCAST [RS_5] PartitionCols:_col2 - Select Operator [SEL_1] (rows=1 width=368) + Select Operator [SEL_2] (rows=1 width=368) Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=1 width=368) - default@t1_n119,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] - <-Select Operator [SEL_3] (rows=1 width=184) + Filter Operator [FIL_17] (rows=1 width=368) + predicate:UDFToDouble(key) is not null + TableScan [TS_0] (rows=1 width=368) + default@t1_n119,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + <-Select Operator [SEL_4] (rows=1 width=184) Output:["_col0","_col1"] - TableScan [TS_2] (rows=1 width=184) + TableScan [TS_3] (rows=1 width=184) default@t2_n70,c,Tbl:COMPLETE,Col:NONE,Output:["key"] PREHOOK: query: explain FROM T1_n119 a FULL OUTER JOIN T2_n70 c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) @@ -6642,27 +6650,29 @@ Stage-0 limit:-1 Stage-1 Reducer 3 llap - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=16) + File Output Operator [FS_15] + Group By Operator [GBY_13] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_11] - Group By Operator [GBY_10] (rows=1 width=16) + PARTITION_ONLY_SHUFFLE [RS_12] + Group By Operator [GBY_11] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_9] + PARTITION_ONLY_SHUFFLE [RS_10] PartitionCols:rand() - Select Operator [SEL_7] (rows=1 width=202) + Select Operator [SEL_8] (rows=1 width=202) Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_17] (rows=1 width=202) - Conds:SEL_1._col1=RS_5._col1(Left Outer),Output:["_col0","_col2"] + Map Join Operator [MAPJOIN_18] (rows=1 width=202) + Conds:SEL_1._col1=RS_6._col1(Left Outer),Output:["_col0","_col2"] <-Map 4 [BROADCAST_EDGE] llap - BROADCAST [RS_5] + BROADCAST [RS_6] PartitionCols:_col1 - Select Operator [SEL_3] (rows=1 width=368) + Select Operator [SEL_4] (rows=1 width=368) Output:["_col0","_col1"] - TableScan [TS_2] (rows=1 width=368) - default@t1_n119,v,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Filter Operator [FIL_17] (rows=1 width=368) + predicate:UDFToDouble(key) is not null + TableScan [TS_2] (rows=1 width=368) + default@t1_n119,v,Tbl:COMPLETE,Col:NONE,Output:["key","val"] <-Select Operator [SEL_1] (rows=1 width=184) Output:["_col0","_col1"] TableScan [TS_0] (rows=1 width=184) diff --git a/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out b/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out index 63c455b3e4..5b2289dd37 100644 --- a/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out +++ b/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out @@ -559,7 +559,7 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 1 Map 3 - Statistics: Num rows: 24737 Data size: 197896 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 18466 Data size: 147728 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() minReductionHashAggr: 0.0 @@ -576,16 +576,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -675,7 +679,7 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 1 Map 3 - Statistics: Num rows: 24737 Data size: 197896 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 18466 Data size: 147728 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() minReductionHashAggr: 0.0 @@ -692,16 +696,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out b/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out index 338f840938..067e42bffe 100644 --- a/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out +++ b/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out @@ -1769,7 +1769,8 @@ INNER JOIN ((SELECT `key`, `value` FROM `default`.`src1` WHERE `value` IS NOT NULL) AS `t2` LEFT JOIN (SELECT `key` -FROM `default`.`src`) AS `t3` ON `t2`.`key` = `t3`.`key`) ON `t0`.`value` = `t2`.`value` +FROM `default`.`src` +WHERE `key` IS NOT NULL) AS `t4` ON `t2`.`key` = `t4`.`key`) ON `t0`.`value` = `t2`.`value` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 @@ -1869,20 +1870,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - tag: 1 - auto parallelism: true + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + tag: 1 + auto parallelism: true Execution mode: vectorized, llap LLAP IO: no inputs Path -> Alias: diff --git a/ql/src/test/results/clientpositive/llap/join46.q.out b/ql/src/test/results/clientpositive/llap/join46.q.out index ec584299c5..0267f2da78 100644 --- a/ql/src/test/results/clientpositive/llap/join46.q.out +++ b/ql/src/test/results/clientpositive/llap/join46.q.out @@ -89,17 +89,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n0 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -112,10 +116,10 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -202,10 +206,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n0 - filterExpr: key BETWEEN 100 AND 102 (type: boolean) + filterExpr: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) + predicate: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), value (type: int), col_2 (type: string) @@ -427,17 +431,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test1_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map 3 @@ -467,10 +475,10 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1031,17 +1039,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n0 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -1055,14 +1067,14 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1608,17 +1620,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test1_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map 3 @@ -1649,14 +1665,14 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1265 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2162,7 +2178,7 @@ NULL NULL None NULL NULL NULL 101 2 Car 102 2 Del 101 2 Car 103 2 Ema NULL NULL NULL 104 3 Fli -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM ( @@ -2226,55 +2242,61 @@ STAGE PLANS: Map 1 Map Operator Tree: TableScan - alias: test1_n2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + alias: test2_n0 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) + expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: boolean) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map 5 Map Operator Tree: TableScan - alias: test2_n0 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + alias: test1_n2 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) + expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: boolean) - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -2282,20 +2304,20 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Right Outer Join 0 to 1 + Left Outer Join 0 to 1 keys: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + residual filter predicates: {(_col4 BETWEEN 100 AND 102 or _col3)} + Statistics: Num rows: 7 Data size: 1261 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + expressions: _col4 (type: int), _col5 (type: int), _col6 (type: string), _col0 (type: int), _col1 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1233 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1233 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Reducer 3 Execution mode: llap @@ -2308,10 +2330,10 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 63 Data size: 23764 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 63 Data size: 23764 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2327,14 +2349,14 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Stage: Stage-0 @@ -2343,7 +2365,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1_n2.key AS key1, test1_n2.value AS value1, test1_n2.col_1 AS col_1, diff --git a/ql/src/test/results/clientpositive/llap/join_emit_interval.q.out b/ql/src/test/results/clientpositive/llap/join_emit_interval.q.out index 05424ad04d..38bede3ddd 100644 --- a/ql/src/test/results/clientpositive/llap/join_emit_interval.q.out +++ b/ql/src/test/results/clientpositive/llap/join_emit_interval.q.out @@ -89,17 +89,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n4 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -115,14 +119,14 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out b/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out index b0e8aeaf08..c07ee83216 100644 --- a/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out @@ -47,17 +47,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src2 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -152,17 +156,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src2 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -254,16 +262,19 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -366,16 +377,19 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 @@ -388,16 +402,19 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -528,16 +545,19 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 @@ -550,16 +570,19 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -707,16 +730,19 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 @@ -733,16 +759,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src3 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -963,17 +993,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src2 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -1069,17 +1103,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src2 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -1173,16 +1211,19 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -1286,16 +1327,19 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 @@ -1309,16 +1353,19 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -1450,16 +1497,19 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 @@ -1473,16 +1523,19 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -1633,16 +1686,19 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 @@ -1659,16 +1715,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src3 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/lineage2.q.out b/ql/src/test/results/clientpositive/llap/lineage2.q.out index 9543864b2e..c0cd91f53d 100644 --- a/ql/src/test/results/clientpositive/llap/lineage2.q.out +++ b/ql/src/test/results/clientpositive/llap/lineage2.q.out @@ -552,7 +552,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@src1 PREHOOK: Output: default@dest_l1_n0 -{"version":"1.0","engine":"tez","database":"default","hash":"40b5d904f13549d8c25bd0be758f5b6f","queryText":"INSERT OVERWRITE TABLE dest_l1_n0\nSELECT j.*\nFROM (SELECT t1.key, p1.value\n FROM src1 t1\n LEFT OUTER JOIN src p1\n ON (t1.key = p1.key)\n UNION ALL\n SELECT t2.key, p2.value\n FROM src1 t2\n LEFT OUTER JOIN src p2\n ON (t2.key = p2.key)) j","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(j.key)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"j.value","edgeType":"PROJECTION"},{"sources":[4,2],"targets":[0,1],"expression":"(p1.key = t1.key)","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(p2.key = t2.key)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0],"expression":"compute_stats(UDFToInteger(j.key), 'hll')","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"compute_stats(j.value, 'hll')","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1_n0.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1_n0.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src.key"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"40b5d904f13549d8c25bd0be758f5b6f","queryText":"INSERT OVERWRITE TABLE dest_l1_n0\nSELECT j.*\nFROM (SELECT t1.key, p1.value\n FROM src1 t1\n LEFT OUTER JOIN src p1\n ON (t1.key = p1.key)\n UNION ALL\n SELECT t2.key, p2.value\n FROM src1 t2\n LEFT OUTER JOIN src p2\n ON (t2.key = p2.key)) j","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(j.key)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"j.value","edgeType":"PROJECTION"},{"sources":[4],"targets":[0,1],"expression":"p1.key is not null","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(p1.key = t1.key)","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"p2.key is not null","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(p2.key = t2.key)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0],"expression":"compute_stats(UDFToInteger(j.key), 'hll')","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"compute_stats(j.value, 'hll')","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1_n0.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1_n0.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src.key"}]} PREHOOK: query: drop table if exists emp PREHOOK: type: DROPTABLE PREHOOK: query: drop table if exists dept_n10 diff --git a/ql/src/test/results/clientpositive/llap/lineage3.q.out b/ql/src/test/results/clientpositive/llap/lineage3.q.out index b9ef761f49..783dfa7c13 100644 --- a/ql/src/test/results/clientpositive/llap/lineage3.q.out +++ b/ql/src/test/results/clientpositive/llap/lineage3.q.out @@ -180,13 +180,13 @@ PREHOOK: Input: default@src1 #### A masked pattern was here #### {"version":"1.0","engine":"tez","database":"default","hash":"94e9cc0a67801fe1503a3cb0c5029d59","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0D)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"UDFToDouble((UDFToInteger(b.ctinyint) + 300)) is not null","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 311 val_311 -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select key, value from src1 where key not in (select key+18 from src1) order by key PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"tez","database":"default","hash":"cbc4367150807328dda0f1cf4c74b811","queryText":"select key, value from src1\nwhere key not in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + 18.0D))","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"((true is null or (count(*) = 0L)) and (src1.key is not null or (count(*) = 0L) or true is not null) and ((count((UDFToDouble(src1.key) + 18.0D)) >= count(*)) or (count(*) = 0L) or true is not null or src1.key is null))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"TABLE","vertexId":"default.src1"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"cbc4367150807328dda0f1cf4c74b811","queryText":"select key, value from src1\nwhere key not in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + 18.0D))","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"UDFToDouble(src1.key) is not null","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"((true is null or (count(*) = 0L)) and (src1.key is not null or (count(*) = 0L) or true is not null) and ((count((UDFToDouble(src1.key) + 18.0D)) >= count(*)) or (count(*) = 0L) or true is not null or src1.key is null))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"TABLE","vertexId":"default.src1"}]} PREHOOK: query: select * from src1 a where not exists (select cint from alltypesorc b diff --git a/ql/src/test/results/clientpositive/llap/mapjoin3.q.out b/ql/src/test/results/clientpositive/llap/mapjoin3.q.out index ac36e4ff44..39af0d707e 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin3.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin3.q.out @@ -147,17 +147,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test_1 + filterExpr: member is not null (type: boolean) Statistics: Num rows: 3 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: member (type: bigint), age (type: varchar(100)), age (type: varchar(100)) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) + Filter Operator + predicate: member is not null (type: boolean) + Statistics: Num rows: 3 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: member (type: bigint), age (type: varchar(100)), age (type: varchar(100)) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: varchar(100)), _col2 (type: varchar(100)) + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 3 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(100)), _col2 (type: varchar(100)) Execution mode: vectorized, llap LLAP IO: no inputs diff --git a/ql/src/test/results/clientpositive/llap/mapjoin46.q.out b/ql/src/test/results/clientpositive/llap/mapjoin46.q.out index d9d239611b..fe648f6226 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin46.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin46.q.out @@ -86,10 +86,10 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: 1 Map 2 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -100,17 +100,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs @@ -213,10 +217,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n2 - filterExpr: key BETWEEN 100 AND 102 (type: boolean) + filterExpr: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) + predicate: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), value (type: int), col_2 (type: string) @@ -408,17 +412,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test1_n4 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs Map 2 @@ -439,10 +447,10 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: 0 Map 1 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -983,14 +991,14 @@ STAGE PLANS: input vertices: 1 Map 2 residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1001,17 +1009,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs @@ -1424,17 +1436,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test1_n4 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs Map 2 @@ -1456,14 +1472,14 @@ STAGE PLANS: input vertices: 0 Map 1 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1265 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2442,7 +2458,7 @@ POSTHOOK: Input: default@test2_n2 NULL NULL NULL 104 3 Fli NULL NULL NULL 105 NULL None NULL NULL None NULL NULL NULL -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM ( @@ -2498,30 +2514,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Map 3 <- Map 1 (BROADCAST_EDGE) + Map 1 <- Map 3 (BROADCAST_EDGE) + Map 3 <- Map 4 (BROADCAST_EDGE) Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: test1_n4 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + alias: test2_n2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) + expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -2530,46 +2536,59 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 4 - residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 3 + residual filter predicates: {(_col4 BETWEEN 100 AND 102 or _col3)} + Statistics: Num rows: 7 Data size: 1261 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) + expressions: _col4 (type: int), _col5 (type: int), _col6 (type: string), _col0 (type: int), _col1 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1233 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1233 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Execution mode: llap LLAP IO: no inputs Map 3 Map Operator Tree: TableScan - alias: test2_n2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + alias: test1_n4 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) + expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: - Right Outer Join 0 to 1 + Left Outer Join 0 to 1 keys: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 0 Map 1 - residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 4 + residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Execution mode: llap LLAP IO: no inputs @@ -2577,17 +2596,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -2601,10 +2624,10 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 63 Data size: 23764 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 63 Data size: 23764 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2616,7 +2639,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1_n4.key AS key1, test1_n4.value AS value1, test1_n4.col_1 AS col_1, @@ -2681,7 +2704,7 @@ NULL NULL NULL 105 NULL None 98 NULL None NULL NULL NULL NULL NULL NULL 105 NULL None 99 0 Alice NULL NULL NULL NULL NULL NULL 105 NULL None 99 2 Mat 102 2 Del NULL NULL NULL 105 NULL None NULL NULL None NULL NULL NULL -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM ( @@ -2737,30 +2760,20 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 4 (BROADCAST_EDGE) - Map 3 <- Map 1 (BROADCAST_EDGE) + Map 1 <- Map 3 (BROADCAST_EDGE) + Map 3 <- Map 4 (BROADCAST_EDGE) Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: test1_n4 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + alias: test2_n2 + Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) + expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -2769,46 +2782,59 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 4 - residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 3 + residual filter predicates: {(_col4 BETWEEN 100 AND 102 or _col3)} + Statistics: Num rows: 7 Data size: 1261 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) + expressions: _col4 (type: int), _col5 (type: int), _col6 (type: string), _col0 (type: int), _col1 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1233 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1233 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Execution mode: llap LLAP IO: no inputs Map 3 Map Operator Tree: TableScan - alias: test2_n2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE + alias: test1_n4 + Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string), key BETWEEN 100 AND 102 (type: boolean) + expressions: key (type: int), value (type: int), col_1 (type: string), key BETWEEN 100 AND 102 (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 596 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: - Right Outer Join 0 to 1 + Left Outer Join 0 to 1 keys: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 0 Map 1 - residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + 1 Map 4 + residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Execution mode: llap LLAP IO: no inputs @@ -2816,17 +2842,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -2840,10 +2870,10 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 63 Data size: 23764 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 63 Data size: 23764 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2855,7 +2885,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1_n4.key AS key1, test1_n4.value AS value1, test1_n4.col_1 AS col_1, diff --git a/ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out b/ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out index 8c9008a3d8..3cb69a836e 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin_emit_interval.q.out @@ -89,14 +89,14 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6 input vertices: 1 Map 2 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -107,17 +107,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs diff --git a/ql/src/test/results/clientpositive/llap/mergejoin.q.out b/ql/src/test/results/clientpositive/llap/mergejoin.q.out index 61d50b0c27..6872f9d3b3 100644 --- a/ql/src/test/results/clientpositive/llap/mergejoin.q.out +++ b/ql/src/test/results/clientpositive/llap/mergejoin.q.out @@ -1537,11 +1537,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Map Operator Tree: TableScan alias: a @@ -1671,11 +1675,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Map Operator Tree: TableScan alias: b diff --git a/ql/src/test/results/clientpositive/llap/sharedwork.q.out b/ql/src/test/results/clientpositive/llap/sharedwork.q.out index 09c10b8a11..3721a0a275 100644 --- a/ql/src/test/results/clientpositive/llap/sharedwork.q.out +++ b/ql/src/test/results/clientpositive/llap/sharedwork.q.out @@ -198,6 +198,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: table__1232 + filterExpr: ((col_24) IN ('part1', 'part2', 'part3') or col_24 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator @@ -217,19 +218,23 @@ STAGE PLANS: tag: 1 value expressions: _col1 (type: string) auto parallelism: true - Select Operator - expressions: col_24 (type: string), col_21 (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + isSamplingPred: false + predicate: col_24 is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: col_24 (type: string), col_21 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: _col1 (type: string) - auto parallelism: true + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: _col1 (type: string) + auto parallelism: true Execution mode: vectorized, llap LLAP IO: no inputs Path -> Alias: @@ -289,21 +294,26 @@ STAGE PLANS: Map Operator Tree: TableScan alias: table__133 + filterExpr: col_1 is not null (type: boolean) Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: col_1 (type: string), col_22 (type: string), col_23 (type: int) - outputColumnNames: _col0, _col1, _col2 + Filter Operator + isSamplingPred: false + predicate: col_1 is not null (type: boolean) Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: col_1 (type: string), col_22 (type: string), col_23 (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE - tag: 1 - value expressions: _col1 (type: string), _col2 (type: int) - auto parallelism: true + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: _col1 (type: string), _col2 (type: int) + auto parallelism: true Execution mode: vectorized, llap LLAP IO: no inputs Path -> Alias: @@ -363,12 +373,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: table__1215 - filterExpr: (UDFToDouble(col_100) = 210.0D) (type: boolean) + filterExpr: ((UDFToDouble(col_100) = 210.0D) and col_1 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (UDFToDouble(col_100) = 210.0D) (type: boolean) + predicate: ((UDFToDouble(col_100) = 210.0D) and col_1 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: col_1 (type: string) diff --git a/ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out b/ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out index 35f7051ebe..ec296ecc2b 100644 --- a/ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out +++ b/ql/src/test/results/clientpositive/llap/skewjoinopt15.q.out @@ -203,17 +203,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 6 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), val (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 6 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), val (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map 3 @@ -425,16 +429,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 4 diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out index 544ca58541..57a252d95a 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_17.q.out @@ -636,96 +636,120 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 4 Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 5 Map Operator Tree: TableScan alias: d + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 6 Map Operator Tree: TableScan alias: e + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 7 Map Operator Tree: TableScan alias: f + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 8 Map Operator Tree: TableScan alias: g + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -928,112 +952,140 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 4 Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 5 Map Operator Tree: TableScan alias: d + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 6 Map Operator Tree: TableScan alias: e + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 7 Map Operator Tree: TableScan alias: f + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 8 Map Operator Tree: TableScan alias: g + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 9 Map Operator Tree: TableScan alias: h + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -1383,304 +1435,380 @@ STAGE PLANS: Map Operator Tree: TableScan alias: j + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 11 Map Operator Tree: TableScan alias: k + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 12 Map Operator Tree: TableScan alias: l + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 13 Map Operator Tree: TableScan alias: m + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 14 Map Operator Tree: TableScan alias: n + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 15 Map Operator Tree: TableScan alias: o + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 16 Map Operator Tree: TableScan alias: p + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 17 Map Operator Tree: TableScan alias: q + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 18 Map Operator Tree: TableScan alias: r + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 19 Map Operator Tree: TableScan alias: s + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 2 Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 20 Map Operator Tree: TableScan alias: t + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 3 Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 4 Map Operator Tree: TableScan alias: d + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 5 Map Operator Tree: TableScan alias: e + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 6 Map Operator Tree: TableScan alias: f + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 7 Map Operator Tree: TableScan alias: g + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 8 Map Operator Tree: TableScan alias: h + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 9 Map Operator Tree: TableScan alias: i + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out index cf0663ed6c..ce3bc247d2 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_4.q.out @@ -324,11 +324,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: a @@ -356,17 +360,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -441,33 +449,41 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) Execution mode: llap Map 4 Map Operator Tree: @@ -559,11 +575,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: a @@ -808,11 +828,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: b @@ -840,17 +864,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 3 @@ -924,33 +952,41 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator - condition map: - Right Outer Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) Execution mode: llap Map 4 Map Operator Tree: @@ -1042,11 +1078,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: b @@ -1323,17 +1363,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -1422,45 +1466,50 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Map 2 Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized, llap - LLAP IO: no inputs - Map 4 Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized, llap - LLAP IO: no inputs - Map 5 + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) + Execution mode: llap + Map 4 Map Operator Tree: TableScan alias: c @@ -1477,23 +1526,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) Reducer 3 Execution mode: llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out index fa997c4fd6..df458ba2d0 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_5.q.out @@ -324,11 +324,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: a @@ -356,17 +360,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -441,33 +449,41 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) Execution mode: llap Map 4 Map Operator Tree: @@ -559,11 +575,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: a @@ -808,11 +828,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: b @@ -840,17 +864,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 3 @@ -924,33 +952,41 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator - condition map: - Right Outer Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) Execution mode: llap Map 4 Map Operator Tree: @@ -1042,11 +1078,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: b @@ -1323,17 +1363,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -1422,45 +1466,50 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Map 2 Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized, llap - LLAP IO: no inputs - Map 4 Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized, llap - LLAP IO: no inputs - Map 5 + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) + Execution mode: llap + Map 4 Map Operator Tree: TableScan alias: c @@ -1477,23 +1526,6 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Full Outer Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string) Reducer 3 Execution mode: llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out b/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out index a905927023..b5326999c6 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out @@ -1638,9 +1638,9 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@src_null_n4 POSTHOOK: Lineage: src_null_n4.key SCRIPT [] POSTHOOK: Lineage: src_null_n4.value EXPRESSION [] -Warning: Map Join MAPJOIN[131][bigTable=?] in task 'Map 1' is a cross product -Warning: Map Join MAPJOIN[132][bigTable=?] in task 'Map 5' is a cross product -Warning: Map Join MAPJOIN[133][bigTable=?] in task 'Reducer 6' is a cross product +Warning: Map Join MAPJOIN[133][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[134][bigTable=?] in task 'Map 5' is a cross product +Warning: Map Join MAPJOIN[135][bigTable=?] in task 'Reducer 6' is a cross product PREHOOK: query: explain select key, value, count(*) from src_null_n4 b @@ -1757,40 +1757,44 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: value is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3 - input vertices: - 1 Reducer 4 - residual filter predicates: {(_col2 <> _col1)} - Statistics: Num rows: 1 Data size: 553 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col3 is null (type: boolean) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + input vertices: + 1 Reducer 4 + residual filter predicates: {(_col2 <> _col1)} Statistics: Num rows: 1 Data size: 553 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: _col3 is null (type: boolean) Statistics: Num rows: 1 Data size: 553 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: string), _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash + Select Operator + expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 553 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string) + Group By Operator + keys: _col1 (type: string), _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 553 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 553 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -1992,9 +1996,9 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[131][bigTable=?] in task 'Map 1' is a cross product -Warning: Map Join MAPJOIN[132][bigTable=?] in task 'Map 5' is a cross product -Warning: Map Join MAPJOIN[133][bigTable=?] in task 'Reducer 6' is a cross product +Warning: Map Join MAPJOIN[133][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[134][bigTable=?] in task 'Map 5' is a cross product +Warning: Map Join MAPJOIN[135][bigTable=?] in task 'Reducer 6' is a cross product PREHOOK: query: select key, value, count(*) from src_null_n4 b where NOT EXISTS (select key from src_null_n4 where src_null_n4.value <> b.value) diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out index a6b8885ef4..10858c2515 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -260,7 +260,7 @@ POSTHOOK: Input: default@part_null 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product +Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -312,17 +312,20 @@ STAGE PLANS: TableScan alias: part_null Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Filter Operator + predicate: p_name is not null (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Group By Operator + keys: p_name (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_name (type: string) outputColumnNames: p_name @@ -465,7 +468,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product +Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -474,7 +477,7 @@ POSTHOOK: query: select * from part_null where p_name IN (select p_name from par POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product +Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -526,17 +529,20 @@ STAGE PLANS: TableScan alias: part_null Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_type (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Filter Operator + predicate: p_type is not null (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Group By Operator + keys: p_type (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_type (type: string) outputColumnNames: p_type @@ -687,7 +693,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product +Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -947,7 +953,7 @@ POSTHOOK: query: select * from part_null where p_brand IN (select p_brand from p POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -1000,17 +1006,20 @@ STAGE PLANS: TableScan alias: tempty Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c (type: char(2)) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Filter Operator + predicate: CAST( c AS STRING) is not null (type: boolean) Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: char(2)) - sort order: + - Map-reduce partition columns: _col0 (type: char(2)) + Group By Operator + keys: c (type: char(2)) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: char(2)) + sort order: + + Map-reduce partition columns: _col0 (type: char(2)) + Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c (type: char(2)) outputColumnNames: c @@ -1151,7 +1160,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -3043,7 +3052,7 @@ POSTHOOK: Input: default@part_null 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join MERGEJOIN[83][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product +Warning: Shuffle Join MERGEJOIN[85][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product PREHOOK: query: explain select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -3262,16 +3271,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (_col0 / _col1) (type: double), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) + Filter Operator + predicate: (_col0 is not null and _col1 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (_col0 / _col1) (type: double), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) Select Operator expressions: (_col0 / _col1) (type: double) outputColumnNames: _col0 @@ -3292,7 +3304,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[83][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product +Warning: Shuffle Join MERGEJOIN[85][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product PREHOOK: query: select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -4054,7 +4066,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tnull POSTHOOK: Lineage: tnull.c SCRIPT [] POSTHOOK: Lineage: tnull.i SCRIPT [] -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where p_partkey = 3 OR p_size NOT IN (select i from tnull) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4102,17 +4114,20 @@ STAGE PLANS: TableScan alias: tnull Statistics: Num rows: 3 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: i (type: int) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 + Filter Operator + predicate: i is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Group By Operator + keys: i (type: int) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: i (type: int) outputColumnNames: i @@ -4210,7 +4225,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from part where p_partkey = 3 OR p_size NOT IN (select i from tnull) PREHOOK: type: QUERY PREHOOK: Input: default@part diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index c21810cf07..e55396f83f 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -591,7 +591,7 @@ Manufacturer#4 almond azure aquamarine papaya violet 12 Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#5 almond aquamarine dodger light gainsboro 46 Manufacturer#5 almond azure blanched chiffon midnight 23 -Warning: Shuffle Join MERGEJOIN[46][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[48][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select p_name, p_size from @@ -795,16 +795,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (UDFToDouble(_col0) / _col1) (type: double), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) + Filter Operator + predicate: (UDFToDouble(_col0) is not null and _col1 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (UDFToDouble(_col0) / _col1) (type: double), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) Reducer 7 Execution mode: vectorized, llap Reduce Operator Tree: @@ -833,7 +836,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[48][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[50][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select p_name, p_size from part where part.p_size not in @@ -1837,7 +1840,7 @@ POSTHOOK: Input: default@part 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_partkey PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -1973,17 +1976,20 @@ STAGE PLANS: expressions: _col1 (type: int) outputColumnNames: _col1 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col1 (type: int) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col1 (type: int) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col1) minReductionHashAggr: 0.0 @@ -2031,7 +2037,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_partkey PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2056,7 +2062,7 @@ POSTHOOK: Input: default@part 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2179,17 +2185,20 @@ STAGE PLANS: expressions: _col1 (type: int) outputColumnNames: _col1 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col1 (type: int) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col1 (type: int) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col1) minReductionHashAggr: 0.0 @@ -2237,7 +2246,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2512,7 +2521,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### 26 -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2631,21 +2640,32 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: double) + outputColumnNames: _col1 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: floor(_col1) (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: bigint) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: floor(_col1) (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col0) minReductionHashAggr: 0.0 @@ -2693,7 +2713,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3557,7 +3577,7 @@ almond aquamarine sandy cyan gainsboro almond aquamarine yellow dodger mint almond azure aquamarine papaya violet almond azure blanched chiffon midnight -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -3606,17 +3626,20 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string) outputColumnNames: key @@ -3805,7 +3828,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -4146,7 +4169,7 @@ POSTHOOK: Input: default@src 5 3 70 3 90 3 -Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4282,17 +4305,20 @@ STAGE PLANS: expressions: _col1 (type: int) outputColumnNames: _col1 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col1 (type: int) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col1 (type: int) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col1) minReductionHashAggr: 0.0 @@ -4340,7 +4366,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[40][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4365,7 +4391,7 @@ POSTHOOK: Input: default@part 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4505,17 +4531,20 @@ STAGE PLANS: expressions: _col1 (type: int) outputColumnNames: _col1 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col1 (type: int) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col1 (type: int) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col1) minReductionHashAggr: 0.0 @@ -4563,7 +4592,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4576,7 +4605,7 @@ POSTHOOK: Input: default@part 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously 65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull -Warning: Shuffle Join MERGEJOIN[45][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[47][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4648,44 +4677,52 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part + filterExpr: p_name is not null (type: boolean) Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: p_name is not null (type: boolean) Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - minReductionHashAggr: 0.0 - mode: hash + Select Operator + expressions: p_name (type: string) outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 7 Map Operator Tree: TableScan alias: part + filterExpr: p_brand is not null (type: boolean) Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: p_brand is not null (type: boolean) Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - minReductionHashAggr: 0.0 - mode: hash + Select Operator + expressions: p_brand (type: string) outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 8 @@ -4794,7 +4831,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[45][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[47][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -5390,10 +5427,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e - filterExpr: UDFToDouble((p_size + 100)) is not null (type: boolean) + filterExpr: (UDFToDouble((p_size + 100)) is not null and p_size is not null) (type: boolean) Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: UDFToDouble((p_size + 100)) is not null (type: boolean) + predicate: (UDFToDouble((p_size + 100)) is not null and p_size is not null) (type: boolean) Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_size (type: int) @@ -5581,7 +5618,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@t2_n0 POSTHOOK: Lineage: t2_n0.c1 SCRIPT [] -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain SELECT c1 FROM t1_n0 WHERE c1 NOT IN (SELECT c1 FROM t2_n0) PREHOOK: type: QUERY PREHOOK: Input: default@t1_n0 @@ -5628,17 +5665,20 @@ STAGE PLANS: TableScan alias: t2_n0 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: c1 (type: int) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: c1 (type: int) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: c1 (type: int) outputColumnNames: c1 @@ -5736,7 +5776,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: SELECT c1 FROM t1_n0 WHERE c1 NOT IN (SELECT c1 FROM t2_n0) PREHOOK: type: QUERY PREHOOK: Input: default@t1_n0 @@ -5830,11 +5870,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1_n0 - filterExpr: UDFToDouble(c2) is not null (type: boolean) + filterExpr: (UDFToDouble(c2) is not null and c2 is not null) (type: boolean) Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: UDFToDouble(c2) is not null (type: boolean) - Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (UDFToDouble(c2) is not null and c2 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: c2 (type: char(100)) minReductionHashAggr: 0.0 @@ -6929,7 +6969,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@t_n0 #### A masked pattern was here #### 7 -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select t_n0.i from t_n0 where t_n0.j NOT IN (select t1_n0.i from t_n0 t1_n0 ) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 @@ -6975,17 +7015,20 @@ STAGE PLANS: TableScan alias: t1_n0 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: i (type: int) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: i is not null (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: i (type: int) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: i (type: int) outputColumnNames: i @@ -7083,7 +7126,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select t_n0.i from t_n0 where t_n0.j NOT IN (select t1_n0.i from t_n0 t1_n0 ) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 @@ -7094,7 +7137,7 @@ POSTHOOK: Input: default@t_n0 #### A masked pattern was here #### 1 4 -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select t_n0.i from t_n0 where t_n0.i NOT IN (select t1_n0.i from t_n0 t1_n0 ) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 @@ -7123,17 +7166,20 @@ STAGE PLANS: TableScan alias: t1_n0 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: i (type: int) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: i is not null (type: boolean) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: i (type: int) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: i (type: int) outputColumnNames: _col0 @@ -7240,7 +7286,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select t_n0.i from t_n0 where t_n0.i NOT IN (select t1_n0.i from t_n0 t1_n0 ) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 @@ -7253,7 +7299,7 @@ PREHOOK: query: drop table t1_n0 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table t1_n0 POSTHOOK: type: DROPTABLE -Warning: Shuffle Join MERGEJOIN[53][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[54][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from src b where b.key not in @@ -7337,18 +7383,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: value (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -7475,7 +7525,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[53][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[54][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select * from src b where b.key not in diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out index c4f8307f6d..23b5344f15 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out @@ -3439,8 +3439,8 @@ POSTHOOK: Input: default@part 85768 86428 90681 -Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 6' is a cross product -Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[44][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from part_null_n0 where p_name NOT LIKE (select min(p_name) from part_null_n0) AND p_brand NOT IN (select p_name from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3473,17 +3473,20 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_name (type: string) outputColumnNames: p_name @@ -3647,8 +3650,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 6' is a cross product -Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[44][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select * from part_null_n0 where p_name NOT LIKE (select min(p_name) from part_null_n0) AND p_brand NOT IN (select p_name from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3684,7 +3687,7 @@ POSTHOOK: Input: default@part_null_n0 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from part_null_n0 where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null_n0 pp where part_null_n0.p_type = pp.p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3717,17 +3720,20 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_name (type: string) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_name (type: string) outputColumnNames: p_name @@ -3911,7 +3917,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select * from part_null_n0 where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null_n0 pp where part_null_n0.p_type = pp.p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -6241,7 +6247,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from part where p_size <> (select count(p_size) from part pp where part.p_type <> pp.p_type) PREHOOK: type: QUERY @@ -6303,18 +6309,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part + filterExpr: p_type is not null (type: boolean) Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -6409,7 +6419,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select * from part where p_size <> (select count(p_size) from part pp where part.p_type <> pp.p_type) PREHOOK: type: QUERY @@ -6463,7 +6473,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@t_n11 POSTHOOK: Lineage: t_n11.i SCRIPT [] POSTHOOK: Lineage: t_n11.j SCRIPT [] -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from t_n11 where 0 = (select count(*) from t_n11 tt_n11 where tt_n11.j <> t_n11.i) PREHOOK: type: QUERY PREHOOK: Input: default@t_n11 @@ -6523,18 +6533,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t_n11 + filterExpr: i is not null (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: i (type: int) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: i is not null (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: i (type: int) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -6629,7 +6643,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[33][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select * from t_n11 where 0 = (select count(*) from t_n11 tt_n11 where tt_n11.j <> t_n11.i) PREHOOK: type: QUERY PREHOOK: Input: default@t_n11 diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out b/ql/src/test/results/clientpositive/llap/subquery_select.q.out index 0a01852b9f..e8f489e7df 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain SELECT p_size, p_size IN ( SELECT MAX(p_size) FROM part) FROM part @@ -116,16 +116,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) Reducer 5 Execution mode: vectorized, llap Reduce Operator Tree: @@ -154,7 +157,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: SELECT p_size, p_size IN ( SELECT MAX(p_size) FROM part) FROM part @@ -400,7 +403,7 @@ POSTHOOK: Input: default@part 6 true 6 true 7 true -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain SELECT p_size, p_size NOT IN ( SELECT MAX(p_size) FROM part) FROM part @@ -518,16 +521,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) Reducer 5 Execution mode: vectorized, llap Reduce Operator Tree: @@ -556,7 +562,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: SELECT p_size, p_size NOT IN ( SELECT MAX(p_size) FROM part) FROM part @@ -2226,7 +2232,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tnull_n3 POSTHOOK: Lineage: tnull_n3.i EXPRESSION [] -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select p_size, p_size IN (select i from tnull_n3) from part PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2274,17 +2280,20 @@ STAGE PLANS: TableScan alias: tnull_n3 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: i (type: int) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 + Filter Operator + predicate: i is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Group By Operator + keys: i (type: int) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: i (type: int) outputColumnNames: i @@ -2379,7 +2388,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select p_size, p_size IN (select i from tnull_n3) from part PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3193,7 +3202,7 @@ STANDARD ANODIZED TIN true STANDARD BURNISHED TIN true STANDARD PLATED TIN true STANDARD POLISHED STEEL true -Warning: Shuffle Join MERGEJOIN[73][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[75][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 5' is a cross product PREHOOK: query: EXPLAIN SELECT p_size, p_size IN ( SELECT MAX(p_size) FROM part p where p.p_type = part.p_type) AND p_name IN (SELECT min(p_name) from part) @@ -3404,16 +3413,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) Group By Operator aggregations: count(), count(_col0) mode: complete @@ -3434,7 +3446,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[73][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[75][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 5' is a cross product PREHOOK: query: SELECT p_size, p_size IN ( SELECT MAX(p_size) FROM part p where p.p_type = part.p_type) AND p_name IN (SELECT min(p_name) from part) @@ -3620,8 +3632,8 @@ POSTHOOK: Input: default@part 6 false 6 false 7 false -Warning: Shuffle Join MERGEJOIN[50][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[53][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[52][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[55][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select p_size, (p_size IN (select (select max(p_size) from part) as sb from part order by sb limit 1)) = true from part @@ -3773,16 +3785,19 @@ STAGE PLANS: Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) Group By Operator aggregations: count(), count(_col0) minReductionHashAggr: 0.0 @@ -3816,8 +3831,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[50][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[53][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[52][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[55][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select p_size, (p_size IN (select (select max(p_size) from part) as sb from part order by sb limit 1)) = true from part @@ -4275,7 +4290,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### true -Warning: Shuffle Join MERGEJOIN[52][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[53][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select o.p_size, (select count(distinct p_type) from part p where p.p_partkey = o.p_partkey) tmp FROM part o right join (select * from part where p_size > (select avg(p_size) from part)) t on t.p_partkey = o.p_partkey PREHOOK: type: QUERY @@ -4307,16 +4322,19 @@ STAGE PLANS: TableScan alias: o Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: p_partkey is not null (type: boolean) Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) Select Operator expressions: p_partkey (type: int), UDFToDouble(p_size) (type: double) outputColumnNames: _col0, _col1 @@ -4477,7 +4495,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[52][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[53][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select o.p_size, (select count(distinct p_type) from part p where p.p_partkey = o.p_partkey) tmp FROM part o right join (select * from part where p_size > (select avg(p_size) from part)) t on t.p_partkey = o.p_partkey PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out index 8caf6c712a..4866a42b7d 100644 --- a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out @@ -52,16 +52,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: UDFToInteger(key) is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(key) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: UDFToInteger(key) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -164,16 +168,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: UDFToInteger(key) is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(key) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: UDFToInteger(key) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -269,16 +277,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: UDFToInteger(key) is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(key) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: UDFToInteger(key) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: UDFToInteger(key) (type: int) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/tez_fixed_bucket_pruning.q.out b/ql/src/test/results/clientpositive/llap/tez_fixed_bucket_pruning.q.out index 1574565408..e3bebbc6d5 100644 --- a/ql/src/test/results/clientpositive/llap/tez_fixed_bucket_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_fixed_bucket_pruning.q.out @@ -468,7 +468,7 @@ FROM (SELECT `t4`.`plan_key`, `t6`.`project_key` FROM (SELECT `t3`.`l3_snapshot_number`, `t2`.`plan_detail_object_id`, `t0`.`l3_snapshot_number` AS `l3_snapshot_number0`, `t0`.`plan_key`, `t0`.`finplan_detail_object_id` FROM (SELECT `l3_snapshot_number`, `plan_key`, `finplan_detail_object_id` FROM `default`.`l3_monthly_dw_dimplan` -WHERE `idp_data_date` = DATE '2017-12-28') AS `t0` +WHERE `idp_data_date` = DATE '2017-12-28' AND `finplan_detail_object_id` IS NOT NULL AND `l3_snapshot_number` IS NOT NULL) AS `t0` RIGHT JOIN ((SELECT `plan_detail_object_id` FROM `default`.`l3_clarity__l3_monthly_dw_factplan_dw_stg_2018022300104_1` WHERE `project_object_id` = 7147200) AS `t2`, @@ -476,7 +476,7 @@ WHERE `project_object_id` = 7147200) AS `t2`, FROM `default`.`l3_clarity__l3_snap_number_2018022300104`) AS `t3`) ON `t0`.`finplan_detail_object_id` = `t2`.`plan_detail_object_id` AND `t0`.`l3_snapshot_number` = `t3`.`l3_snapshot_number`) AS `t4` LEFT JOIN (SELECT `project_key`, `l3_snapshot_number` FROM `default`.`l3_clarity__l3_monthly_dw_factplan_datajoin_1_s2_2018022300104_1` -WHERE `idp_data_date` = DATE '2017-12-28' AND `project_object_id` = 7147200) AS `t6` ON `t4`.`l3_snapshot_number` = `t6`.`l3_snapshot_number` +WHERE `idp_data_date` = DATE '2017-12-28' AND `project_object_id` = 7147200 AND `l3_snapshot_number` IS NOT NULL) AS `t6` ON `t4`.`l3_snapshot_number` = `t6`.`l3_snapshot_number` ORDER BY `t4`.`plan_key`, `t6`.`project_key` LIMIT 5) AS `t9` STAGE DEPENDENCIES: @@ -691,12 +691,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s1 - filterExpr: (idp_data_date = DATE'2017-12-28') (type: boolean) + filterExpr: ((idp_data_date = DATE'2017-12-28') and finplan_detail_object_id is not null and l3_snapshot_number is not null) (type: boolean) Statistics: Num rows: 180340 Data size: 14427200 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (idp_data_date = DATE'2017-12-28') (type: boolean) + predicate: ((idp_data_date = DATE'2017-12-28') and finplan_detail_object_id is not null and l3_snapshot_number is not null) (type: boolean) Statistics: Num rows: 90170 Data size: 7213600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: l3_snapshot_number (type: bigint), plan_key (type: bigint), finplan_detail_object_id (type: bigint) @@ -772,12 +772,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s2 - filterExpr: ((idp_data_date = DATE'2017-12-28') and (project_object_id = 7147200L)) (type: boolean) + filterExpr: ((idp_data_date = DATE'2017-12-28') and (project_object_id = 7147200L) and l3_snapshot_number is not null) (type: boolean) Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: ((idp_data_date = DATE'2017-12-28') and (project_object_id = 7147200L)) (type: boolean) + predicate: ((idp_data_date = DATE'2017-12-28') and (project_object_id = 7147200L) and l3_snapshot_number is not null) (type: boolean) Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: project_key (type: bigint), l3_snapshot_number (type: bigint) @@ -977,7 +977,7 @@ FROM (SELECT `t4`.`plan_key`, `t6`.`project_key` FROM (SELECT `t3`.`l3_snapshot_number`, `t2`.`plan_detail_object_id`, `t0`.`l3_snapshot_number` AS `l3_snapshot_number0`, `t0`.`plan_key`, `t0`.`finplan_detail_object_id` FROM (SELECT `l3_snapshot_number`, `plan_key`, `finplan_detail_object_id` FROM `default`.`l3_monthly_dw_dimplan` -WHERE `idp_data_date` = DATE '2017-12-28') AS `t0` +WHERE `idp_data_date` = DATE '2017-12-28' AND `finplan_detail_object_id` IS NOT NULL AND `l3_snapshot_number` IS NOT NULL) AS `t0` RIGHT JOIN ((SELECT `plan_detail_object_id` FROM `default`.`l3_clarity__l3_monthly_dw_factplan_dw_stg_2018022300104_1` WHERE `project_object_id` = 7147200) AS `t2`, @@ -985,7 +985,7 @@ WHERE `project_object_id` = 7147200) AS `t2`, FROM `default`.`l3_clarity__l3_snap_number_2018022300104`) AS `t3`) ON `t0`.`finplan_detail_object_id` = `t2`.`plan_detail_object_id` AND `t0`.`l3_snapshot_number` = `t3`.`l3_snapshot_number`) AS `t4` LEFT JOIN (SELECT `project_key`, `l3_snapshot_number` FROM `default`.`l3_clarity__l3_monthly_dw_factplan_datajoin_1_s2_2018022300104_1` -WHERE `idp_data_date` = DATE '2017-12-28' AND `project_object_id` = 7147200) AS `t6` ON `t4`.`l3_snapshot_number` = `t6`.`l3_snapshot_number` +WHERE `idp_data_date` = DATE '2017-12-28' AND `project_object_id` = 7147200 AND `l3_snapshot_number` IS NOT NULL) AS `t6` ON `t4`.`l3_snapshot_number` = `t6`.`l3_snapshot_number` ORDER BY `t4`.`plan_key`, `t6`.`project_key` LIMIT 5) AS `t9` STAGE DEPENDENCIES: @@ -1200,13 +1200,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s1 - filterExpr: (idp_data_date = DATE'2017-12-28') (type: boolean) - buckets included: [50,] of 64 + filterExpr: ((idp_data_date = DATE'2017-12-28') and finplan_detail_object_id is not null and l3_snapshot_number is not null) (type: boolean) Statistics: Num rows: 180340 Data size: 14427200 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: (idp_data_date = DATE'2017-12-28') (type: boolean) + predicate: ((idp_data_date = DATE'2017-12-28') and finplan_detail_object_id is not null and l3_snapshot_number is not null) (type: boolean) Statistics: Num rows: 90170 Data size: 7213600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: l3_snapshot_number (type: bigint), plan_key (type: bigint), finplan_detail_object_id (type: bigint) @@ -1282,12 +1281,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s2 - filterExpr: ((idp_data_date = DATE'2017-12-28') and (project_object_id = 7147200L)) (type: boolean) + filterExpr: ((idp_data_date = DATE'2017-12-28') and (project_object_id = 7147200L) and l3_snapshot_number is not null) (type: boolean) Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false Filter Operator isSamplingPred: false - predicate: ((idp_data_date = DATE'2017-12-28') and (project_object_id = 7147200L)) (type: boolean) + predicate: ((idp_data_date = DATE'2017-12-28') and (project_object_id = 7147200L) and l3_snapshot_number is not null) (type: boolean) Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: project_key (type: bigint), l3_snapshot_number (type: bigint) diff --git a/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out b/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out index 0bddcf7b9b..230cd249ce 100644 --- a/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out @@ -29,16 +29,19 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 @@ -55,16 +58,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -72,17 +79,17 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Right Outer Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 64 Data size: 7120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 39 Data size: 6942 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 64 Data size: 7120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 39 Data size: 6942 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Reducer 3 Execution mode: llap @@ -94,11 +101,11 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 604 Data size: 124027 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 563 Data size: 111606 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 604 Data size: 124027 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 563 Data size: 111606 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) Reducer 4 Execution mode: vectorized, llap @@ -106,10 +113,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 604 Data size: 124027 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 563 Data size: 111606 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 604 Data size: 124027 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 563 Data size: 111606 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out b/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out index d97ad532ba..79c56401fe 100644 --- a/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out @@ -29,16 +29,19 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 @@ -55,16 +58,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -72,17 +79,17 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Right Outer Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 64 Data size: 7120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 39 Data size: 6942 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 64 Data size: 7120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 39 Data size: 6942 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) Reducer 3 Execution mode: llap @@ -94,11 +101,11 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 604 Data size: 124027 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 563 Data size: 111606 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 604 Data size: 124027 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 563 Data size: 111606 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) Reducer 4 Execution mode: vectorized, llap @@ -106,10 +113,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 604 Data size: 124027 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 563 Data size: 111606 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 604 Data size: 124027 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 563 Data size: 111606 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/tez_nway_join.q.out b/ql/src/test/results/clientpositive/llap/tez_nway_join.q.out index 74d76ecc22..48be4275ed 100644 --- a/ql/src/test/results/clientpositive/llap/tez_nway_join.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_nway_join.q.out @@ -348,32 +348,40 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs Map 5 Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -487,32 +495,40 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs Map 4 Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out b/ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out index 771d8f9ff9..57f8f33f81 100644 --- a/ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_smb_empty.q.out @@ -247,11 +247,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s3 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: key (type: int), value (type: string), ds (type: string) - outputColumnNames: _col0, _col1, _col2 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: int), value (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL Map Operator Tree: TableScan alias: s1 diff --git a/ql/src/test/results/clientpositive/llap/tez_smb_reduce_side.q.out b/ql/src/test/results/clientpositive/llap/tez_smb_reduce_side.q.out index 90326ca7b2..9bbb181468 100644 --- a/ql/src/test/results/clientpositive/llap/tez_smb_reduce_side.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_smb_reduce_side.q.out @@ -87,20 +87,24 @@ STAGE PLANS: Map Operator Tree: TableScan alias: x + filterExpr: key is not null (type: boolean) Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs Map 3 diff --git a/ql/src/test/results/clientpositive/llap/tez_union.q.out b/ql/src/test/results/clientpositive/llap/tez_union.q.out index 5224b2495c..bae4176664 100644 --- a/ql/src/test/results/clientpositive/llap/tez_union.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_union.q.out @@ -1295,23 +1295,27 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Union 2 @@ -1355,32 +1359,40 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 3 Map Operator Tree: TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 4 diff --git a/ql/src/test/results/clientpositive/llap/transitive_not_null.q.out b/ql/src/test/results/clientpositive/llap/transitive_not_null.q.out new file mode 100644 index 0000000000..a0089ac631 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/transitive_not_null.q.out @@ -0,0 +1,351 @@ +PREHOOK: query: create table tbl_1(i1 int, j1 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tbl_1 +POSTHOOK: query: create table tbl_1(i1 int, j1 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tbl_1 +PREHOOK: query: insert into tbl_1 values(1,2),(1,null), (null, 200), (45,68) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@tbl_1 +POSTHOOK: query: insert into tbl_1 values(1,2),(1,null), (null, 200), (45,68) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@tbl_1 +POSTHOOK: Lineage: tbl_1.i1 SCRIPT [] +POSTHOOK: Lineage: tbl_1.j1 SCRIPT [] +PREHOOK: query: create table tbl_2(i2 int, j2 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tbl_2 +POSTHOOK: query: create table tbl_2(i2 int, j2 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tbl_2 +PREHOOK: query: insert into tbl_2 values(1,2),(1,null), (null, 200), (45,68) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@tbl_2 +POSTHOOK: query: insert into tbl_2 values(1,2),(1,null), (null, 200), (45,68) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@tbl_2 +POSTHOOK: Lineage: tbl_2.i2 SCRIPT [] +POSTHOOK: Lineage: tbl_2.j2 SCRIPT [] +PREHOOK: query: explain cbo select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: explain cbo select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +CBO PLAN: +HiveJoin(condition=[=($0, $2)], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(i1=[$0], j1=[$1]) + HiveTableScan(table=[[default, tbl_1]], table:alias=[tbl_1]) + HiveProject(i2=[$0], j2=[$1]) + HiveFilter(condition=[IS NOT NULL($0)]) + HiveTableScan(table=[[default, tbl_2]], table:alias=[tbl_2]) + +PREHOOK: query: select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +NULL 200 NULL NULL +1 2 1 2 +1 2 1 NULL +1 NULL 1 2 +1 NULL 1 NULL +45 68 45 68 +PREHOOK: query: explain cbo select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: explain cbo select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +CBO PLAN: +HiveJoin(condition=[=($0, $2)], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(i1=[$0], j1=[$1]) + HiveFilter(condition=[IS NOT NULL($0)]) + HiveTableScan(table=[[default, tbl_1]], table:alias=[tbl_1]) + HiveProject(i2=[$0], j2=[$1]) + HiveTableScan(table=[[default, tbl_2]], table:alias=[tbl_2]) + +PREHOOK: query: select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +NULL NULL NULL 200 +1 2 1 2 +1 2 1 NULL +1 NULL 1 2 +1 NULL 1 NULL +45 68 45 68 +PREHOOK: query: explain cbo select * from tbl_1 full outer join tbl_2 on tbl_1.i1 = tbl_2.i2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: explain cbo select * from tbl_1 full outer join tbl_2 on tbl_1.i1 = tbl_2.i2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +CBO PLAN: +HiveJoin(condition=[=($0, $2)], joinType=[full], algorithm=[none], cost=[not available]) + HiveProject(i1=[$0], j1=[$1]) + HiveTableScan(table=[[default, tbl_1]], table:alias=[tbl_1]) + HiveProject(i2=[$0], j2=[$1]) + HiveTableScan(table=[[default, tbl_2]], table:alias=[tbl_2]) + +PREHOOK: query: select * from tbl_1 full outer join tbl_2 on tbl_1.i1 = tbl_2.i2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from tbl_1 full outer join tbl_2 on tbl_1.i1 = tbl_2.i2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +NULL NULL NULL 200 +NULL 200 NULL NULL +1 2 1 2 +1 2 1 NULL +1 NULL 1 2 +1 NULL 1 NULL +45 68 45 68 +PREHOOK: query: explain cbo select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: explain cbo select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +CBO PLAN: +HiveJoin(condition=[AND(=($0, $2), =($1, $3))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(i1=[$0], j1=[$1]) + HiveTableScan(table=[[default, tbl_1]], table:alias=[tbl_1]) + HiveProject(i2=[$0], j2=[$1]) + HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) + HiveTableScan(table=[[default, tbl_2]], table:alias=[tbl_2]) + +PREHOOK: query: select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +NULL 200 NULL NULL +1 NULL NULL NULL +1 2 1 2 +45 68 45 68 +PREHOOK: query: explain cbo select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: explain cbo select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +CBO PLAN: +HiveJoin(condition=[AND(=($0, $2), =($1, $3))], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(i1=[$0], j1=[$1]) + HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) + HiveTableScan(table=[[default, tbl_1]], table:alias=[tbl_1]) + HiveProject(i2=[$0], j2=[$1]) + HiveTableScan(table=[[default, tbl_2]], table:alias=[tbl_2]) + +PREHOOK: query: select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1=tbl_2.j2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +NULL NULL NULL 200 +NULL NULL 1 NULL +1 2 1 2 +45 68 45 68 +PREHOOK: query: explain cbo select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: explain cbo select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +CBO PLAN: +HiveJoin(condition=[AND(=($0, $2), >($1, $3))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(i1=[$0], j1=[$1]) + HiveTableScan(table=[[default, tbl_1]], table:alias=[tbl_1]) + HiveProject(i2=[$0], j2=[$1]) + HiveFilter(condition=[IS NOT NULL($0)]) + HiveTableScan(table=[[default, tbl_2]], table:alias=[tbl_2]) + +PREHOOK: query: select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from tbl_1 left join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +NULL 200 NULL NULL +1 2 NULL NULL +1 NULL NULL NULL +45 68 NULL NULL +PREHOOK: query: explain cbo select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: explain cbo select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +CBO PLAN: +HiveJoin(condition=[AND(=($0, $2), >($1, $3))], joinType=[right], algorithm=[none], cost=[not available]) + HiveProject(i1=[$0], j1=[$1]) + HiveFilter(condition=[IS NOT NULL($0)]) + HiveTableScan(table=[[default, tbl_1]], table:alias=[tbl_1]) + HiveProject(i2=[$0], j2=[$1]) + HiveTableScan(table=[[default, tbl_2]], table:alias=[tbl_2]) + +PREHOOK: query: select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from tbl_1 right join tbl_2 on tbl_1.i1 = tbl_2.i2 AND tbl_1.j1>tbl_2.j2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +NULL NULL NULL 200 +NULL NULL 1 2 +NULL NULL 1 NULL +NULL NULL 45 68 +PREHOOK: query: explain cbo SELECT t0.col0, t0.col1 +FROM + ( + SELECT i1 as col0, j1 as col1 FROM tbl_1 + ) AS t0 + LEFT JOIN + ( + SELECT i2 as col0, j2 as col1 FROM tbl_2 + ) AS t1 +ON t0.col0 = t1.col0 AND t0.col1 = t1.col1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: explain cbo SELECT t0.col0, t0.col1 +FROM + ( + SELECT i1 as col0, j1 as col1 FROM tbl_1 + ) AS t0 + LEFT JOIN + ( + SELECT i2 as col0, j2 as col1 FROM tbl_2 + ) AS t1 +ON t0.col0 = t1.col0 AND t0.col1 = t1.col1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +CBO PLAN: +HiveProject(col0=[$0], col1=[$1]) + HiveJoin(condition=[AND(=($0, $2), =($1, $3))], joinType=[left], algorithm=[none], cost=[not available]) + HiveProject(i1=[$0], j1=[$1]) + HiveTableScan(table=[[default, tbl_1]], table:alias=[tbl_1]) + HiveProject(i2=[$0], j2=[$1]) + HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))]) + HiveTableScan(table=[[default, tbl_2]], table:alias=[tbl_2]) + +PREHOOK: query: SELECT t0.col0, t0.col1 +FROM + ( + SELECT i1 as col0, j1 as col1 FROM tbl_1 + ) AS t0 + LEFT JOIN + ( + SELECT i2 as col0, j2 as col1 FROM tbl_2 + ) AS t1 +ON t0.col0 = t1.col0 AND t0.col1 = t1.col1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tbl_1 +PREHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT t0.col0, t0.col1 +FROM + ( + SELECT i1 as col0, j1 as col1 FROM tbl_1 + ) AS t0 + LEFT JOIN + ( + SELECT i2 as col0, j2 as col1 FROM tbl_2 + ) AS t1 +ON t0.col0 = t1.col0 AND t0.col1 = t1.col1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Input: default@tbl_2 +#### A masked pattern was here #### +NULL 200 +1 NULL +1 2 +45 68 +PREHOOK: query: DROP TABLE tbl_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tbl_1 +PREHOOK: Output: default@tbl_1 +POSTHOOK: query: DROP TABLE tbl_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tbl_1 +POSTHOOK: Output: default@tbl_1 +PREHOOK: query: DROP TABLE tbl_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tbl_2 +PREHOOK: Output: default@tbl_2 +POSTHOOK: query: DROP TABLE tbl_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tbl_2 +POSTHOOK: Output: default@tbl_2 diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out index 0a341490e1..e3dabce817 100644 --- a/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out @@ -117,7 +117,7 @@ STAGE PLANS: outputColumnNames: _col0, _col2 input vertices: 1 Map 2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: bigint), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -125,13 +125,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 2] - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -157,31 +157,39 @@ STAGE PLANS: Map Operator Tree: TableScan alias: n + filterExpr: member is not null (type: boolean) Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:member:bigint, 1:attr:bigint, 2:ROW__ID:struct] - Select Operator - expressions: member (type: bigint), CASE WHEN (((attr > 1L) or attr is null)) THEN (attr) ELSE (null) END (type: bigint) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0, 6] - selectExpressions: IfExprColumnNull(col 5:boolean, col 1:bigint, null)(children: ColOrCol(col 3:boolean, col 4:boolean)(children: LongColGreaterLongScalar(col 1:bigint, val 1) -> 3:boolean, IsNull(col 1:bigint) -> 4:boolean) -> 5:boolean, col 1:bigint) -> 6:bigint - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 0:bigint + predicateExpression: SelectColumnIsNotNull(col 0:bigint) + predicate: member is not null (type: boolean) + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: member (type: bigint), CASE WHEN (((attr > 1L) or attr is null)) THEN (attr) ELSE (null) END (type: bigint) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 6:bigint - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + projectedOutputColumnNums: [0, 6] + selectExpressions: IfExprColumnNull(col 5:boolean, col 1:bigint, null)(children: ColOrCol(col 3:boolean, col 4:boolean)(children: LongColGreaterLongScalar(col 1:bigint, val 1) -> 3:boolean, IsNull(col 1:bigint) -> 4:boolean) -> 5:boolean, col 1:bigint) -> 6:bigint + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 0:bigint + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 6:bigint + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out index 258545b6e0..062968f0ab 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_mapjoin.q.out @@ -1,4 +1,4 @@ -Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[35][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: explain vectorization expression select * from src @@ -131,28 +131,35 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Group By Operator - Group By Vectorization: - className: VectorGroupByOperator - groupByMode: HASH - keyExpressions: col 0:string - native: false - vectorProcessingMode: HASH - projectedOutputColumnNums: [] - keys: key (type: string) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Reduce Sink Vectorization: - className: VectorReduceSinkStringOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:string) + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + Group By Vectorization: + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 0:string + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [] + keys: key (type: string) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkStringOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string) outputColumnNames: key @@ -296,7 +303,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[35][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: select * from src where not key in @@ -325,7 +332,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@orcsrc POSTHOOK: Lineage: orcsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[35][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: select * from orcsrc where not key in @@ -342,7 +349,7 @@ order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@orcsrc #### A masked pattern was here #### -Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[35][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: select * from orcsrc where not key in diff --git a/ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out b/ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out index 3040515cb7..c9604183f6 100644 --- a/ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_join_nulls.q.out @@ -316,26 +316,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -374,7 +381,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 input vertices: 0 Map 1 - Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 @@ -382,7 +389,7 @@ STAGE PLANS: className: VectorSelectOperator native: true selectExpressions: VectorUDFAdaptor(hash(_col0,_col1,_col2,_col3)) -> 5:int - Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col0) Group By Vectorization: @@ -480,26 +487,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -538,7 +552,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 input vertices: 0 Map 1 - Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 @@ -546,7 +560,7 @@ STAGE PLANS: className: VectorSelectOperator native: true selectExpressions: VectorUDFAdaptor(hash(_col0,_col1,_col2,_col3)) -> 5:int - Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col0) Group By Vectorization: @@ -644,26 +658,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: value is not null (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator + predicate: value is not null (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -702,7 +723,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 input vertices: 0 Map 1 - Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 @@ -710,7 +731,7 @@ STAGE PLANS: className: VectorSelectOperator native: true selectExpressions: VectorUDFAdaptor(hash(_col0,_col1,_col2,_col3)) -> 5:int - Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col0) Group By Vectorization: @@ -810,25 +831,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkMultiKeyOperator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkMultiKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -867,7 +895,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 input vertices: 0 Map 1 - Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: hash(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 @@ -875,7 +903,7 @@ STAGE PLANS: className: VectorSelectOperator native: true selectExpressions: VectorUDFAdaptor(hash(_col0,_col1,_col2,_col3)) -> 5:int - Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(_col0) Group By Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out b/ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out index 3d1f78c64c..524a75467c 100644 --- a/ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_left_outer_join.q.out @@ -55,7 +55,7 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Map 3 - Statistics: Num rows: 24737 Data size: 86492 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 18466 Data size: 61408 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -64,7 +64,7 @@ STAGE PLANS: 1 _col0 (type: tinyint) input vertices: 1 Map 4 - Statistics: Num rows: 2338217 Data size: 18705736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1302989 Data size: 10423912 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() minReductionHashAggr: 0.0 @@ -90,16 +90,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -115,16 +119,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hd + filterExpr: ctinyint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - sort order: + - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ctinyint is not null (type: boolean) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out b/ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out index 261b470c82..75a1c99d61 100644 --- a/ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_left_outer_join2.q.out @@ -128,14 +128,14 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5 input vertices: 1 Map 2 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -146,17 +146,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: char(2)) + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: char(2)) Execution mode: llap LLAP IO: all inputs @@ -230,14 +234,14 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5 input vertices: 1 Map 2 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -248,17 +252,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: char(2)) + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: char(2)) Execution mode: llap LLAP IO: all inputs @@ -347,7 +355,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5 input vertices: 1 Map 2 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 @@ -355,13 +363,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3] - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -387,30 +395,38 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:rnum:int, 1:c1:int, 2:c2:char(2), 3:ROW__ID:struct] - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [1, 2] - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 1:int + predicateExpression: SelectColumnIsNotNull(col 1:int) + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 2:char(2) - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: char(2)) + projectedOutputColumnNums: [1, 2] + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 1:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 2:char(2) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: char(2)) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -514,7 +530,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5 input vertices: 1 Map 2 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 @@ -522,13 +538,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3] - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -554,30 +570,38 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:rnum:int, 1:c1:int, 2:c2:char(2), 3:ROW__ID:struct] - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [1, 2] - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 1:int + predicateExpression: SelectColumnIsNotNull(col 1:int) + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 2:char(2) - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: char(2)) + projectedOutputColumnNums: [1, 2] + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 1:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 2:char(2) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: char(2)) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -685,7 +709,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5 input vertices: 1 Map 2 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 @@ -693,13 +717,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 5] - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -725,30 +749,38 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:rnum:int, 1:c1:int, 2:c2:char(2), 3:ROW__ID:struct] - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [1, 2] - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 1:int + predicateExpression: SelectColumnIsNotNull(col 1:int) + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 2:char(2) - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: char(2)) + projectedOutputColumnNums: [1, 2] + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 1:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 2:char(2) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: char(2)) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -856,7 +888,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5 input vertices: 1 Map 2 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 @@ -864,13 +896,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 5] - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -896,30 +928,38 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:rnum:int, 1:c1:int, 2:c2:char(2), 3:ROW__ID:struct] - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [1, 2] - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 1:int + predicateExpression: SelectColumnIsNotNull(col 1:int) + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 2:char(2) - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: char(2)) + projectedOutputColumnNums: [1, 2] + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 1:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 2:char(2) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: char(2)) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out index 3973675607..8395b7d36f 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join0.q.out @@ -126,13 +126,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 input vertices: 1 Map 2 - Statistics: Num rows: 12 Data size: 2034 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 12 Data size: 2034 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -158,30 +158,38 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t2 + filterExpr: c is not null (type: boolean) Statistics: Num rows: 6 Data size: 554 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:c:int, 1:v2:string, 2:ROW__ID:struct] - Select Operator - expressions: c (type: int), v2 (type: string) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0, 1] - Statistics: Num rows: 6 Data size: 554 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 0:int + predicateExpression: SelectColumnIsNotNull(col 0:int) + predicate: c is not null (type: boolean) + Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: int), v2 (type: string) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 1:string - Statistics: Num rows: 6 Data size: 554 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 0:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 1:string + Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -254,30 +262,38 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 + filterExpr: a is not null (type: boolean) Statistics: Num rows: 6 Data size: 554 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:v1:string, 1:a:int, 2:ROW__ID:struct] - Select Operator - expressions: v1 (type: string), a (type: int) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0, 1] - Statistics: Num rows: 6 Data size: 554 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 1:int + predicateExpression: SelectColumnIsNotNull(col 1:int) + predicate: a is not null (type: boolean) + Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: v1 (type: string), a (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 0:string - Statistics: Num rows: 6 Data size: 554 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 1:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:string + Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -331,13 +347,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 input vertices: 0 Map 1 - Statistics: Num rows: 12 Data size: 2042 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1484 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 12 Data size: 2042 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1484 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out index 27eb70099b..11d45ee9bc 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out @@ -284,13 +284,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: 1 Map 2 - Statistics: Num rows: 28 Data size: 15472 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 18 Data size: 9762 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 28 Data size: 15472 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 18 Data size: 9762 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -316,30 +316,38 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 15 Data size: 3745 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] - Statistics: Num rows: 15 Data size: 3745 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 2:int + predicateExpression: SelectColumnIsNotNull(col 2:int) + predicate: cint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 0:tinyint, 1:smallint, 3:bigint, 4:float, 5:double, 6:string, 7:string, 8:timestamp, 9:timestamp, 10:boolean, 11:boolean - Statistics: Num rows: 15 Data size: 3745 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + Statistics: Num rows: 10 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 2:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:tinyint, 1:smallint, 3:bigint, 4:float, 5:double, 6:string, 7:string, 8:timestamp, 9:timestamp, 10:boolean, 11:boolean + Statistics: Num rows: 10 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -463,13 +471,13 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Map 2 - Statistics: Num rows: 225 Data size: 884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 150 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 225 Data size: 884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 150 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -495,28 +503,36 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hd + filterExpr: ctinyint is not null (type: boolean) Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0] - Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - sort order: + - Map-reduce partition columns: _col0 (type: tinyint) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 0:tinyint + predicateExpression: SelectColumnIsNotNull(col 0:tinyint) + predicate: ctinyint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE + projectedOutputColumnNums: [0] + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 0:tinyint + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -733,7 +749,7 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Map 3 - Statistics: Num rows: 28 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 18 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -752,7 +768,7 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Map 4 - Statistics: Num rows: 420 Data size: 1664 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 180 Data size: 704 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col0) Group By Vectorization: @@ -796,28 +812,36 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [2] - Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 2:int + predicateExpression: SelectColumnIsNotNull(col 2:int) + predicate: cint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE + projectedOutputColumnNums: [2] + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 2:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -839,28 +863,36 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hd + filterExpr: ctinyint is not null (type: boolean) Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0] - Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - sort order: + - Map-reduce partition columns: _col0 (type: tinyint) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 0:tinyint + predicateExpression: SelectColumnIsNotNull(col 0:tinyint) + predicate: ctinyint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE + projectedOutputColumnNums: [0] + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 0:tinyint + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out index f241158417..0d11ae2fb0 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out @@ -299,7 +299,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 50 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -318,7 +318,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 4 - Statistics: Num rows: 100 Data size: 728 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col1) Group By Vectorization: @@ -362,28 +362,36 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [2] - Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 2:int + predicateExpression: SelectColumnIsNotNull(col 2:int) + predicate: cint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE + projectedOutputColumnNums: [2] + Statistics: Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 2:int + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -405,28 +413,36 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hd + filterExpr: cbigint is not null (type: boolean) Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: cbigint (type: bigint) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [3] - Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator - keyColumns: 3:bigint + predicateExpression: SelectColumnIsNotNull(col 3:bigint) + predicate: cbigint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + projectedOutputColumnNums: [3] + Statistics: Num rows: 10 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + keyColumns: 3:bigint + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 10 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index 5dcf5f1d87..a7db2b3075 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -3449,6 +3449,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) @@ -3560,6 +3561,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) @@ -6568,6 +6570,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) @@ -6656,6 +6659,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 389248 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ds (type: string) diff --git a/ql/src/test/results/clientpositive/llap/vectorized_join46.q.out b/ql/src/test/results/clientpositive/llap/vectorized_join46.q.out index 1b4d3438ef..bb23424ef9 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_join46.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_join46.q.out @@ -102,13 +102,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: 1 Map 2 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -128,26 +128,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n9 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -290,7 +297,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n9 - filterExpr: key BETWEEN 100 AND 102 (type: boolean) + filterExpr: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true @@ -298,7 +305,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicate: key BETWEEN 100 AND 102 (type: boolean) + predicate: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), value (type: int), col_2 (type: string) @@ -565,26 +572,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test1_n14 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -623,13 +637,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 input vertices: 0 Map 1 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1325,14 +1339,14 @@ STAGE PLANS: input vertices: 1 Map 2 residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1349,26 +1363,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test2_n9 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1886,26 +1907,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: test1_n14 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkLongOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: @@ -1936,14 +1964,14 @@ STAGE PLANS: input vertices: 0 Map 1 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1265 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 1237 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/mapjoin1.q.out b/ql/src/test/results/clientpositive/mapjoin1.q.out index 5f381bac2a..269dd40524 100644 --- a/ql/src/test/results/clientpositive/mapjoin1.q.out +++ b/ql/src/test/results/clientpositive/mapjoin1.q.out @@ -47,15 +47,19 @@ STAGE PLANS: $hdt$_0:a TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Stage: Stage-3 Map Reduce @@ -138,18 +142,22 @@ STAGE PLANS: $hdt$_0:a TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 - 1 {_col2} - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + filter predicates: + 0 + 1 {_col2} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Stage: Stage-3 Map Reduce @@ -237,18 +245,22 @@ STAGE PLANS: $hdt$_0:a TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 - 1 {_col2} - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + filter predicates: + 0 + 1 {_col2} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Stage: Stage-3 Map Reduce @@ -336,15 +348,19 @@ STAGE PLANS: $hdt$_0:a TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Stage: Stage-3 Map Reduce @@ -427,18 +443,22 @@ STAGE PLANS: $hdt$_0:a TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 - 1 {_col2} - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + filter predicates: + 0 + 1 {_col2} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Stage: Stage-3 Map Reduce @@ -526,18 +546,22 @@ STAGE PLANS: $hdt$_0:a TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 - 1 {_col2} - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + filter predicates: + 0 + 1 {_col2} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Stage: Stage-3 Map Reduce diff --git a/ql/src/test/results/clientpositive/mapjoin3.q.out b/ql/src/test/results/clientpositive/mapjoin3.q.out index 92d8f60c5f..f333db562f 100644 --- a/ql/src/test/results/clientpositive/mapjoin3.q.out +++ b/ql/src/test/results/clientpositive/mapjoin3.q.out @@ -115,15 +115,19 @@ STAGE PLANS: $hdt$_1:test_1 TableScan alias: test_1 + filterExpr: member is not null (type: boolean) Statistics: Num rows: 3 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: member (type: bigint), age (type: varchar(100)), age (type: varchar(100)) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: bigint) - 1 _col0 (type: bigint) + Filter Operator + predicate: member is not null (type: boolean) + Statistics: Num rows: 3 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: member (type: bigint), age (type: varchar(100)), age (type: varchar(100)) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: bigint) + 1 _col0 (type: bigint) Stage: Stage-3 Map Reduce diff --git a/ql/src/test/results/clientpositive/mapjoin46.q.out b/ql/src/test/results/clientpositive/mapjoin46.q.out index 2837ca1f6c..77f65e5480 100644 --- a/ql/src/test/results/clientpositive/mapjoin46.q.out +++ b/ql/src/test/results/clientpositive/mapjoin46.q.out @@ -72,15 +72,19 @@ STAGE PLANS: $hdt$_1:test2_n2 TableScan alias: test2_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) Stage: Stage-3 Map Reduce @@ -99,10 +103,10 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -174,10 +178,10 @@ STAGE PLANS: $hdt$_1:test2_n2 TableScan alias: test2_n2 - filterExpr: key BETWEEN 100 AND 102 (type: boolean) + filterExpr: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) + predicate: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), value (type: int), col_2 (type: string) @@ -404,15 +408,19 @@ STAGE PLANS: $hdt$_0:test1_n4 TableScan alias: test1_n4 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) Stage: Stage-3 Map Reduce @@ -431,10 +439,10 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -955,15 +963,19 @@ STAGE PLANS: $hdt$_1:test2_n2 TableScan alias: test2_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) Stage: Stage-3 Map Reduce @@ -983,14 +995,14 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1401,15 +1413,19 @@ STAGE PLANS: $hdt$_0:test1_n4 TableScan alias: test1_n4 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) Stage: Stage-3 Map Reduce @@ -1429,14 +1445,14 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2303,7 +2319,7 @@ POSTHOOK: Input: default@test2_n2 NULL NULL NULL 104 3 Fli NULL NULL NULL 105 NULL None NULL NULL None NULL NULL NULL -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM ( @@ -2359,37 +2375,45 @@ STAGE PLANS: Stage: Stage-8 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_0:$hdt$_0:test1_n4 + $hdt$_0:$hdt$_1:test1_n4 Fetch Operator limit: -1 $hdt$_1:$hdt$_2:test2_n2 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_0:$hdt$_0:test1_n4 + $hdt$_0:$hdt$_1:test1_n4 TableScan alias: test1_n4 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) $hdt$_1:$hdt$_2:test2_n2 TableScan alias: test2_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) Stage: Stage-2 Map Reduce @@ -2403,20 +2427,20 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: - Right Outer Join 0 to 1 + Left Outer Join 0 to 1 keys: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + residual filter predicates: {(_col4 BETWEEN 100 AND 102 or _col3)} + Statistics: Num rows: 8 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + expressions: _col4 (type: int), _col5 (type: int), _col6 (type: string), _col0 (type: int), _col1 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) TableScan alias: test1_n4 @@ -2433,14 +2457,14 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Local Work: Map Reduce Local Work @@ -2453,10 +2477,10 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 72 Data size: 27306 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 72 Data size: 27306 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2468,7 +2492,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1_n4.key AS key1, test1_n4.value AS value1, test1_n4.col_1 AS col_1, @@ -2533,7 +2557,7 @@ NULL NULL NULL 105 NULL None 98 NULL None NULL NULL NULL NULL NULL NULL 105 NULL None 99 0 Alice NULL NULL NULL NULL NULL NULL 105 NULL None 99 2 Mat 102 2 Del NULL NULL NULL 105 NULL None NULL NULL None NULL NULL NULL -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM ( @@ -2589,37 +2613,45 @@ STAGE PLANS: Stage: Stage-8 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_0:$hdt$_0:test1_n4 + $hdt$_0:$hdt$_1:test1_n4 Fetch Operator limit: -1 $hdt$_1:$hdt$_2:test2_n2 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_0:$hdt$_0:test1_n4 + $hdt$_0:$hdt$_1:test1_n4 TableScan alias: test1_n4 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) $hdt$_1:$hdt$_2:test2_n2 TableScan alias: test2_n2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) Stage: Stage-2 Map Reduce @@ -2633,20 +2665,20 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: - Right Outer Join 0 to 1 + Left Outer Join 0 to 1 keys: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + residual filter predicates: {(_col4 BETWEEN 100 AND 102 or _col3)} + Statistics: Num rows: 8 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) + expressions: _col4 (type: int), _col5 (type: int), _col6 (type: string), _col0 (type: int), _col1 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) TableScan alias: test1_n4 @@ -2663,14 +2695,14 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) Local Work: Map Reduce Local Work @@ -2683,10 +2715,10 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 residual filter predicates: {(_col1 is null or (_col10 is null and (_col7 <> _col4)))} - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 72 Data size: 27306 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 110 Data size: 41722 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 72 Data size: 27306 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2698,7 +2730,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT * FROM ( SELECT test1_n4.key AS key1, test1_n4.value AS value1, test1_n4.col_1 AS col_1, diff --git a/ql/src/test/results/clientpositive/mapjoin47.q.out b/ql/src/test/results/clientpositive/mapjoin47.q.out index 9cd3b1eadd..f6eafbc0e1 100644 --- a/ql/src/test/results/clientpositive/mapjoin47.q.out +++ b/ql/src/test/results/clientpositive/mapjoin47.q.out @@ -802,7 +802,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-5:MAPRED' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-5:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -842,15 +842,19 @@ STAGE PLANS: $hdt$_1:b TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) $hdt$_2:a TableScan alias: a @@ -914,7 +918,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-5:MAPRED' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-5:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src JOIN src1 a ON (a.key+src.key >= 100) @@ -943,7 +947,7 @@ POSTHOOK: Input: default@src1 238 val_238 273 val_273 238 val_238 238 val_238 224 238 val_238 238 val_238 369 238 val_238 -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-5:MAPRED' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-5:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -983,15 +987,19 @@ STAGE PLANS: $hdt$_1:a TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) $hdt$_2:b TableScan alias: b @@ -1055,7 +1063,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-5:MAPRED' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-5:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src LEFT OUTER JOIN src1 a ON (a.key = src.key) @@ -1084,7 +1092,7 @@ POSTHOOK: Input: default@src1 4 val_4 NULL NULL 66 val_66 35 val_35 NULL NULL 66 val_66 12 val_12 NULL NULL 66 val_66 -Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[29][bigTable=?] in task 'Stage-7:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -1140,30 +1148,34 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - residual filter predicates: {((_col5 + _col2) >= 100.0D)} - Statistics: Num rows: 4166 Data size: 1537254 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4166 Data size: 1470598 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + residual filter predicates: {((_col5 + _col2) >= 100.0D)} + Statistics: Num rows: 4166 Data size: 1537254 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4166 Data size: 1470598 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -1265,7 +1277,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[29][bigTable=?] in task 'Stage-7:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src JOIN src1 a ON (a.key+src.key >= 100) @@ -1294,7 +1306,7 @@ POSTHOOK: Input: default@src1 238 val_238 273 val_273 238 val_238 238 val_238 224 238 val_238 238 val_238 369 238 val_238 -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-5:MAPRED' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-5:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM src @@ -1334,15 +1346,19 @@ STAGE PLANS: $hdt$_0:src TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) $hdt$_2:b TableScan alias: b @@ -1406,7 +1422,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-5:MAPRED' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-5:MAPRED' is a cross product PREHOOK: query: SELECT * FROM src RIGHT OUTER JOIN src1 a ON (a.key = src.key) @@ -1729,11 +1745,11 @@ POSTHOOK: Input: default@src1 11 val_11 NULL NULL 66 val_66 12 val_12 NULL NULL 66 val_66 12 val_12 NULL NULL 66 val_66 -Warning: Map Join MAPJOIN[40][bigTable=?] in task 'Stage-8:MAPRED' is a cross product -Warning: Map Join MAPJOIN[49][bigTable=?] in task 'Stage-9:MAPRED' is a cross product -Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product -Warning: Map Join MAPJOIN[50][bigTable=?] in task 'Stage-11:MAPRED' is a cross product -Warning: Map Join MAPJOIN[51][bigTable=?] in task 'Stage-11:MAPRED' is a cross product +Warning: Map Join MAPJOIN[42][bigTable=?] in task 'Stage-8:MAPRED' is a cross product +Warning: Map Join MAPJOIN[51][bigTable=?] in task 'Stage-9:MAPRED' is a cross product +Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[52][bigTable=?] in task 'Stage-11:MAPRED' is a cross product +Warning: Map Join MAPJOIN[53][bigTable=?] in task 'Stage-11:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM cbo_t1 t1 @@ -1872,11 +1888,11 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 residual filter predicates: {((_col0 = _col25) or ((_col29 = _col17) and _col6))} {((_col13 = _col25) or ((_col28 = _col16) and (_col14 = _col26)))} {((_col7 = _col25) or ((_col8 = _col26) and (_col27 = _col21)))} - Statistics: Num rows: 337500 Data size: 618975000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 303750 Data size: 557077500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col19 (type: string), _col20 (type: string), _col21 (type: int), _col22 (type: float), _col23 (type: boolean), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: float), _col29 (type: boolean), _col30 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: int), _col10 (type: float), _col11 (type: boolean), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: int), _col16 (type: float), _col17 (type: boolean), _col18 (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29 - Statistics: Num rows: 337500 Data size: 617625000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 303750 Data size: 555862500 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 18300 Basic stats: COMPLETE Column stats: COMPLETE @@ -1916,11 +1932,11 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 residual filter predicates: {((_col0 = _col25) or ((_col29 = _col17) and _col6))} {((_col13 = _col25) or ((_col28 = _col16) and (_col14 = _col26)))} {((_col7 = _col25) or ((_col8 = _col26) and (_col27 = _col21)))} - Statistics: Num rows: 337500 Data size: 618975000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 303750 Data size: 557077500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col19 (type: string), _col20 (type: string), _col21 (type: int), _col22 (type: float), _col23 (type: boolean), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: float), _col29 (type: boolean), _col30 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: int), _col10 (type: float), _col11 (type: boolean), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: int), _col16 (type: float), _col17 (type: boolean), _col18 (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29 - Statistics: Num rows: 337500 Data size: 617625000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 303750 Data size: 555862500 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 18300 Basic stats: COMPLETE Column stats: COMPLETE @@ -1945,7 +1961,7 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 100 Data size: 72921 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 90 Data size: 65601 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: int), _col9 (type: float), _col10 (type: boolean), _col11 (type: string) Reduce Operator Tree: Join Operator @@ -1956,11 +1972,11 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30 residual filter predicates: {((_col0 = _col25) or ((_col29 = _col17) and _col6))} {((_col13 = _col25) or ((_col28 = _col16) and (_col14 = _col26)))} {((_col7 = _col25) or ((_col8 = _col26) and (_col27 = _col21)))} - Statistics: Num rows: 337500 Data size: 618975000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 303750 Data size: 557077500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col19 (type: string), _col20 (type: string), _col21 (type: int), _col22 (type: float), _col23 (type: boolean), _col24 (type: string), _col25 (type: string), _col26 (type: string), _col27 (type: int), _col28 (type: float), _col29 (type: boolean), _col30 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: int), _col10 (type: float), _col11 (type: boolean), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: int), _col16 (type: float), _col17 (type: boolean), _col18 (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: boolean), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29 - Statistics: Num rows: 337500 Data size: 617625000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 303750 Data size: 555862500 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 18300 Basic stats: COMPLETE Column stats: COMPLETE @@ -1982,15 +1998,19 @@ STAGE PLANS: $hdt$_3:$hdt$_3:t1 TableScan alias: t1 + filterExpr: key is not null (type: boolean) Statistics: Num rows: 20 Data size: 7138 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string), c_int (type: int), c_float (type: float), c_boolean (type: boolean), dt (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 20 Data size: 7138 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 18 Data size: 6406 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), c_int (type: int), c_float (type: float), c_boolean (type: boolean), dt (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 18 Data size: 6406 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Stage: Stage-13 Map Reduce @@ -2009,7 +2029,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 100 Data size: 72921 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 90 Data size: 65601 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false table: @@ -2026,11 +2046,11 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[40][bigTable=?] in task 'Stage-8:MAPRED' is a cross product -Warning: Map Join MAPJOIN[49][bigTable=?] in task 'Stage-9:MAPRED' is a cross product -Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product -Warning: Map Join MAPJOIN[50][bigTable=?] in task 'Stage-11:MAPRED' is a cross product -Warning: Map Join MAPJOIN[51][bigTable=?] in task 'Stage-11:MAPRED' is a cross product +Warning: Map Join MAPJOIN[42][bigTable=?] in task 'Stage-8:MAPRED' is a cross product +Warning: Map Join MAPJOIN[51][bigTable=?] in task 'Stage-9:MAPRED' is a cross product +Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[52][bigTable=?] in task 'Stage-11:MAPRED' is a cross product +Warning: Map Join MAPJOIN[53][bigTable=?] in task 'Stage-11:MAPRED' is a cross product PREHOOK: query: SELECT * FROM cbo_t1 t1 RIGHT OUTER JOIN cbo_t1 t2 ON (t2.key = t1.key) diff --git a/ql/src/test/results/clientpositive/masking_10.q.out b/ql/src/test/results/clientpositive/masking_10.q.out index d6ce930e26..7cf86f3db1 100644 --- a/ql/src/test/results/clientpositive/masking_10.q.out +++ b/ql/src/test/results/clientpositive/masking_10.q.out @@ -87,7 +87,7 @@ POSTHOOK: Input: default@masking_test 2017 2_lav 2017 4_lav 2017 8_lav -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from masking_test alias01 diff --git a/ql/src/test/results/clientpositive/mergejoin.q.out b/ql/src/test/results/clientpositive/mergejoin.q.out index 05850d4b0a..baf9079dd5 100644 --- a/ql/src/test/results/clientpositive/mergejoin.q.out +++ b/ql/src/test/results/clientpositive/mergejoin.q.out @@ -1423,16 +1423,20 @@ STAGE PLANS: Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE Map Vectorization: enabled: false enabledConditionsNotMet: Vectorized map work only works with 1 TableScanOperator IS false @@ -1567,16 +1571,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: b Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/mergejoins_mixed.q.out b/ql/src/test/results/clientpositive/mergejoins_mixed.q.out index 31feb8da94..5b8afffadc 100644 --- a/ql/src/test/results/clientpositive/mergejoins_mixed.q.out +++ b/ql/src/test/results/clientpositive/mergejoins_mixed.q.out @@ -449,17 +449,21 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) TableScan alias: d + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -565,21 +569,25 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string) TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: - Right Outer Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col2 (type: string) 1 _col0 (type: string) @@ -720,17 +728,21 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -836,10 +848,10 @@ STAGE PLANS: value expressions: _col1 (type: string) TableScan alias: b - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -879,21 +891,25 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: - Right Outer Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col3 (type: string) 1 _col0 (type: string) @@ -1189,17 +1205,21 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1344,17 +1364,21 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1456,10 +1480,10 @@ STAGE PLANS: value expressions: _col1 (type: string) TableScan alias: b - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -1538,17 +1562,21 @@ STAGE PLANS: value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) TableScan alias: d + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1611,10 +1639,10 @@ STAGE PLANS: value expressions: _col1 (type: string) TableScan alias: b - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -1809,17 +1837,21 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -1848,17 +1880,21 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: string) TableScan alias: d + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: diff --git a/ql/src/test/results/clientpositive/optional_outer.q.out b/ql/src/test/results/clientpositive/optional_outer.q.out index ad82211a0a..7050c6aacc 100644 --- a/ql/src/test/results/clientpositive/optional_outer.q.out +++ b/ql/src/test/results/clientpositive/optional_outer.q.out @@ -29,17 +29,21 @@ STAGE PLANS: value expressions: _col1 (type: string) TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -94,17 +98,21 @@ STAGE PLANS: value expressions: _col1 (type: string) TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -146,17 +154,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) TableScan alias: b Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE @@ -211,17 +223,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) TableScan alias: b Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/perf/spark/query40.q.out b/ql/src/test/results/clientpositive/perf/spark/query40.q.out index 8692d6a2fb..9ec434e735 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query40.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query40.q.out @@ -154,10 +154,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: catalog_returns - filterExpr: cr_item_sk is not null (type: boolean) + filterExpr: (cr_order_number is not null and cr_item_sk is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: cr_item_sk is not null (type: boolean) + predicate: (cr_item_sk is not null and cr_order_number is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cr_item_sk (type: int), cr_order_number (type: int), cr_refunded_cash (type: decimal(7,2)) diff --git a/ql/src/test/results/clientpositive/perf/spark/query45.q.out b/ql/src/test/results/clientpositive/perf/spark/query45.q.out index 4c6f4c2a93..6252f23a60 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query45.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query45.q.out @@ -135,10 +135,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: item - filterExpr: (i_item_sk) IN (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) (type: boolean) + filterExpr: ((i_item_sk) IN (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) and i_item_id is not null) (type: boolean) Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (i_item_sk) IN (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) (type: boolean) + predicate: ((i_item_sk) IN (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) and i_item_id is not null) (type: boolean) Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: i_item_id (type: string) diff --git a/ql/src/test/results/clientpositive/perf/spark/query5.q.out b/ql/src/test/results/clientpositive/perf/spark/query5.q.out index af8bf12ecb..4ecc6bb965 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query5.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query5.q.out @@ -449,7 +449,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: web_sales - filterExpr: (ws_web_site_sk is not null and ws_item_sk is not null and ws_order_number is not null) (type: boolean) + filterExpr: (ws_item_sk is not null and ws_order_number is not null and ws_web_site_sk is not null) (type: boolean) Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (ws_item_sk is not null and ws_order_number is not null and ws_web_site_sk is not null) (type: boolean) diff --git a/ql/src/test/results/clientpositive/perf/spark/query72.q.out b/ql/src/test/results/clientpositive/perf/spark/query72.q.out index 55b720a54c..66de773c1b 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query72.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query72.q.out @@ -138,15 +138,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: promotion + filterExpr: p_promo_sk is not null (type: boolean) Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_promo_sk (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: p_promo_sk is not null (type: boolean) Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col5 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: p_promo_sk (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col5 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -299,10 +303,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: catalog_returns - filterExpr: cr_item_sk is not null (type: boolean) + filterExpr: (cr_item_sk is not null and cr_order_number is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: cr_item_sk is not null (type: boolean) + predicate: (cr_item_sk is not null and cr_order_number is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cr_item_sk (type: int), cr_order_number (type: int) diff --git a/ql/src/test/results/clientpositive/perf/spark/query75.q.out b/ql/src/test/results/clientpositive/perf/spark/query75.q.out index d5bd3d286d..c3b51e6fa6 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query75.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query75.q.out @@ -232,10 +232,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: catalog_returns - filterExpr: cr_item_sk is not null (type: boolean) + filterExpr: (cr_order_number is not null and cr_item_sk is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: cr_item_sk is not null (type: boolean) + predicate: (cr_item_sk is not null and cr_order_number is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cr_item_sk (type: int), cr_order_number (type: int), cr_return_quantity (type: int), cr_return_amount (type: decimal(7,2)) @@ -291,10 +291,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: store_returns - filterExpr: sr_item_sk is not null (type: boolean) + filterExpr: (sr_ticket_number is not null and sr_item_sk is not null) (type: boolean) Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: sr_item_sk is not null (type: boolean) + predicate: (sr_item_sk is not null and sr_ticket_number is not null) (type: boolean) Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sr_item_sk (type: int), sr_ticket_number (type: int), sr_return_quantity (type: int), sr_return_amt (type: decimal(7,2)) @@ -331,10 +331,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: web_returns - filterExpr: wr_item_sk is not null (type: boolean) + filterExpr: (wr_order_number is not null and wr_item_sk is not null) (type: boolean) Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: wr_item_sk is not null (type: boolean) + predicate: (wr_item_sk is not null and wr_order_number is not null) (type: boolean) Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: wr_item_sk (type: int), wr_order_number (type: int), wr_return_quantity (type: int), wr_return_amt (type: decimal(7,2)) @@ -410,10 +410,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: catalog_returns - filterExpr: cr_item_sk is not null (type: boolean) + filterExpr: (cr_order_number is not null and cr_item_sk is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: cr_item_sk is not null (type: boolean) + predicate: (cr_item_sk is not null and cr_order_number is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cr_item_sk (type: int), cr_order_number (type: int), cr_return_quantity (type: int), cr_return_amount (type: decimal(7,2)) @@ -450,10 +450,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: store_returns - filterExpr: sr_item_sk is not null (type: boolean) + filterExpr: (sr_ticket_number is not null and sr_item_sk is not null) (type: boolean) Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: sr_item_sk is not null (type: boolean) + predicate: (sr_item_sk is not null and sr_ticket_number is not null) (type: boolean) Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sr_item_sk (type: int), sr_ticket_number (type: int), sr_return_quantity (type: int), sr_return_amt (type: decimal(7,2)) @@ -490,10 +490,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: web_returns - filterExpr: wr_item_sk is not null (type: boolean) + filterExpr: (wr_order_number is not null and wr_item_sk is not null) (type: boolean) Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: wr_item_sk is not null (type: boolean) + predicate: (wr_item_sk is not null and wr_order_number is not null) (type: boolean) Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: wr_item_sk (type: int), wr_order_number (type: int), wr_return_quantity (type: int), wr_return_amt (type: decimal(7,2)) diff --git a/ql/src/test/results/clientpositive/perf/spark/query78.q.out b/ql/src/test/results/clientpositive/perf/spark/query78.q.out index 6f37cecfe7..1579c4c15c 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query78.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query78.q.out @@ -212,10 +212,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: web_returns - filterExpr: wr_item_sk is not null (type: boolean) + filterExpr: (wr_order_number is not null and wr_item_sk is not null) (type: boolean) Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: wr_item_sk is not null (type: boolean) + predicate: (wr_item_sk is not null and wr_order_number is not null) (type: boolean) Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: wr_item_sk (type: int), wr_order_number (type: int) @@ -270,10 +270,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: catalog_returns - filterExpr: cr_item_sk is not null (type: boolean) + filterExpr: (cr_order_number is not null and cr_item_sk is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: cr_item_sk is not null (type: boolean) + predicate: (cr_item_sk is not null and cr_order_number is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cr_item_sk (type: int), cr_order_number (type: int) @@ -309,10 +309,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: store_returns - filterExpr: sr_item_sk is not null (type: boolean) + filterExpr: (sr_ticket_number is not null and sr_item_sk is not null) (type: boolean) Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: sr_item_sk is not null (type: boolean) + predicate: (sr_item_sk is not null and sr_ticket_number is not null) (type: boolean) Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sr_item_sk (type: int), sr_ticket_number (type: int) diff --git a/ql/src/test/results/clientpositive/perf/spark/query80.q.out b/ql/src/test/results/clientpositive/perf/spark/query80.q.out index b94089bddd..d5aef629a6 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query80.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query80.q.out @@ -476,10 +476,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: catalog_returns - filterExpr: cr_item_sk is not null (type: boolean) + filterExpr: (cr_item_sk is not null and cr_order_number is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: cr_item_sk is not null (type: boolean) + predicate: (cr_item_sk is not null and cr_order_number is not null) (type: boolean) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cr_item_sk (type: int), cr_order_number (type: int), cr_return_amount (type: decimal(7,2)), cr_net_loss (type: decimal(7,2)) @@ -555,10 +555,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: web_returns - filterExpr: wr_item_sk is not null (type: boolean) + filterExpr: (wr_item_sk is not null and wr_order_number is not null) (type: boolean) Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: wr_item_sk is not null (type: boolean) + predicate: (wr_item_sk is not null and wr_order_number is not null) (type: boolean) Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: wr_item_sk (type: int), wr_order_number (type: int), wr_return_amt (type: decimal(7,2)), wr_net_loss (type: decimal(7,2)) @@ -594,10 +594,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: store_returns - filterExpr: sr_item_sk is not null (type: boolean) + filterExpr: (sr_item_sk is not null and sr_ticket_number is not null) (type: boolean) Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: sr_item_sk is not null (type: boolean) + predicate: (sr_item_sk is not null and sr_ticket_number is not null) (type: boolean) Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sr_item_sk (type: int), sr_ticket_number (type: int), sr_return_amt (type: decimal(7,2)), sr_net_loss (type: decimal(7,2)) diff --git a/ql/src/test/results/clientpositive/perf/spark/query93.q.out b/ql/src/test/results/clientpositive/perf/spark/query93.q.out index cf8b6899a4..ed56e6b7cf 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query93.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query93.q.out @@ -83,7 +83,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: store_returns - filterExpr: (sr_reason_sk is not null and sr_item_sk is not null and sr_ticket_number is not null) (type: boolean) + filterExpr: (sr_item_sk is not null and sr_ticket_number is not null and sr_reason_sk is not null) (type: boolean) Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (sr_item_sk is not null and sr_reason_sk is not null and sr_ticket_number is not null) (type: boolean) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query40.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query40.q.out index 9c68f5a9aa..adb6e9cdc1 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query40.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query40.q.out @@ -80,7 +80,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveFilter(condition=[AND(IS NOT NULL($14), IS NOT NULL($15), IS NOT NULL($0))]) HiveTableScan(table=[[default, catalog_sales]], table:alias=[catalog_sales]) HiveProject(cr_item_sk=[$2], cr_order_number=[$16], cr_refunded_cash=[$23]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($16), IS NOT NULL($2))]) HiveTableScan(table=[[default, catalog_returns]], table:alias=[catalog_returns]) HiveProject(d_date_sk=[$0], <=[<(CAST($2):DATE, 1998-04-08)], >==[>=(CAST($2):DATE, 1998-04-08)]) HiveFilter(condition=[AND(BETWEEN(false, CAST($2):TIMESTAMP(9), 1998-03-09 00:00:00:TIMESTAMP(9), 1998-05-08 00:00:00:TIMESTAMP(9)), IS NOT NULL($0))]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query5.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query5.q.out index 7d2af5da42..78d97dc495 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query5.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query5.q.out @@ -328,7 +328,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveProject(wsr_web_site_sk=[$1], date_sk=[$3], sales_price=[CAST(0:DECIMAL(7, 2)):DECIMAL(7, 2)], profit=[CAST(0:DECIMAL(7, 2)):DECIMAL(7, 2)], return_amt=[$6], net_loss=[$7]) HiveJoin(condition=[AND(=($4, $0), =($5, $2))], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(ws_item_sk=[$3], ws_web_site_sk=[$13], ws_order_number=[$17]) - HiveFilter(condition=[AND(IS NOT NULL($13), IS NOT NULL($3), IS NOT NULL($17))]) + HiveFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($17), IS NOT NULL($13))]) HiveTableScan(table=[[default, web_sales]], table:alias=[web_sales]) HiveProject(wr_returned_date_sk=[$0], wr_item_sk=[$2], wr_order_number=[$13], wr_return_amt=[$15], wr_net_loss=[$23]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($2), IS NOT NULL($13))]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query72.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query72.q.out index e49b44bf32..30eec8f39e 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query72.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query72.q.out @@ -120,11 +120,12 @@ HiveSortLimit(sort0=[$5], sort1=[$0], sort2=[$1], sort3=[$2], dir0=[DESC-nulls-l HiveFilter(condition=[AND(=($2, _UTF-16LE'1001-5000'), IS NOT NULL($0))]) HiveTableScan(table=[[default, household_demographics]], table:alias=[household_demographics]) HiveProject(p_promo_sk=[$0]) - HiveTableScan(table=[[default, promotion]], table:alias=[promotion]) + HiveFilter(condition=[IS NOT NULL($0)]) + HiveTableScan(table=[[default, promotion]], table:alias=[promotion]) HiveProject(d_date_sk=[$0], d_week_seq=[$4]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($4))]) HiveTableScan(table=[[default, date_dim]], table:alias=[d2]) HiveProject(cr_item_sk=[$2], cr_order_number=[$16]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($16))]) HiveTableScan(table=[[default, catalog_returns]], table:alias=[catalog_returns]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query75.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query75.q.out index 7bc1cc676f..1374e8d6ab 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query75.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query75.q.out @@ -172,7 +172,7 @@ HiveProject(prev_year=[CAST(2001):INTEGER], year=[CAST(2002):INTEGER], i_brand_i HiveProject(i_brand_id=[$11], i_class_id=[$12], i_category_id=[$13], i_manufact_id=[$14], sales_cnt=[-($7, CASE(IS NOT NULL($2), $2, 0))], sales_amt=[-($8, CASE(IS NOT NULL($3), $3, 0:DECIMAL(1, 0)))]) HiveJoin(condition=[AND(=($6, $1), =($5, $0))], joinType=[right], algorithm=[none], cost=[not available]) HiveProject(cr_item_sk=[$2], cr_order_number=[$16], cr_return_quantity=[$17], cr_return_amount=[$18]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($16), IS NOT NULL($2))]) HiveTableScan(table=[[default, catalog_returns]], table:alias=[catalog_returns]) HiveJoin(condition=[=($6, $1)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available]) @@ -188,7 +188,7 @@ HiveProject(prev_year=[CAST(2001):INTEGER], year=[CAST(2002):INTEGER], i_brand_i HiveProject(i_brand_id=[$11], i_class_id=[$12], i_category_id=[$13], i_manufact_id=[$14], sales_cnt=[-($7, CASE(IS NOT NULL($2), $2, 0))], sales_amt=[-($8, CASE(IS NOT NULL($3), $3, 0:DECIMAL(1, 0)))]) HiveJoin(condition=[AND(=($6, $1), =($5, $0))], joinType=[right], algorithm=[none], cost=[not available]) HiveProject(sr_item_sk=[$2], sr_ticket_number=[$9], sr_return_quantity=[$10], sr_return_amt=[$11]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($9), IS NOT NULL($2))]) HiveTableScan(table=[[default, store_returns]], table:alias=[store_returns]) HiveJoin(condition=[=($6, $1)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available]) @@ -204,7 +204,7 @@ HiveProject(prev_year=[CAST(2001):INTEGER], year=[CAST(2002):INTEGER], i_brand_i HiveProject(i_brand_id=[$11], i_class_id=[$12], i_category_id=[$13], i_manufact_id=[$14], sales_cnt=[-($7, CASE(IS NOT NULL($2), $2, 0))], sales_amt=[-($8, CASE(IS NOT NULL($3), $3, 0:DECIMAL(1, 0)))]) HiveJoin(condition=[AND(=($6, $1), =($5, $0))], joinType=[right], algorithm=[none], cost=[not available]) HiveProject(wr_item_sk=[$2], wr_order_number=[$13], wr_return_quantity=[$14], wr_return_amt=[$15]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($13), IS NOT NULL($2))]) HiveTableScan(table=[[default, web_returns]], table:alias=[web_returns]) HiveJoin(condition=[=($6, $1)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available]) @@ -230,7 +230,7 @@ HiveProject(prev_year=[CAST(2001):INTEGER], year=[CAST(2002):INTEGER], i_brand_i HiveProject(i_brand_id=[$11], i_class_id=[$12], i_category_id=[$13], i_manufact_id=[$14], sales_cnt=[-($7, CASE(IS NOT NULL($2), $2, 0))], sales_amt=[-($8, CASE(IS NOT NULL($3), $3, 0:DECIMAL(1, 0)))]) HiveJoin(condition=[AND(=($6, $1), =($5, $0))], joinType=[right], algorithm=[none], cost=[not available]) HiveProject(cr_item_sk=[$2], cr_order_number=[$16], cr_return_quantity=[$17], cr_return_amount=[$18]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($16), IS NOT NULL($2))]) HiveTableScan(table=[[default, catalog_returns]], table:alias=[catalog_returns]) HiveJoin(condition=[=($6, $1)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available]) @@ -246,7 +246,7 @@ HiveProject(prev_year=[CAST(2001):INTEGER], year=[CAST(2002):INTEGER], i_brand_i HiveProject(i_brand_id=[$11], i_class_id=[$12], i_category_id=[$13], i_manufact_id=[$14], sales_cnt=[-($7, CASE(IS NOT NULL($2), $2, 0))], sales_amt=[-($8, CASE(IS NOT NULL($3), $3, 0:DECIMAL(1, 0)))]) HiveJoin(condition=[AND(=($6, $1), =($5, $0))], joinType=[right], algorithm=[none], cost=[not available]) HiveProject(sr_item_sk=[$2], sr_ticket_number=[$9], sr_return_quantity=[$10], sr_return_amt=[$11]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($9), IS NOT NULL($2))]) HiveTableScan(table=[[default, store_returns]], table:alias=[store_returns]) HiveJoin(condition=[=($6, $1)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available]) @@ -262,7 +262,7 @@ HiveProject(prev_year=[CAST(2001):INTEGER], year=[CAST(2002):INTEGER], i_brand_i HiveProject(i_brand_id=[$11], i_class_id=[$12], i_category_id=[$13], i_manufact_id=[$14], sales_cnt=[-($7, CASE(IS NOT NULL($2), $2, 0))], sales_amt=[-($8, CASE(IS NOT NULL($3), $3, 0:DECIMAL(1, 0)))]) HiveJoin(condition=[AND(=($6, $1), =($5, $0))], joinType=[right], algorithm=[none], cost=[not available]) HiveProject(wr_item_sk=[$2], wr_order_number=[$13], wr_return_quantity=[$14], wr_return_amt=[$15]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($13), IS NOT NULL($2))]) HiveTableScan(table=[[default, web_returns]], table:alias=[web_returns]) HiveJoin(condition=[=($6, $1)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query78.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query78.q.out index 59df50ee57..0c314a6141 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query78.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query78.q.out @@ -148,7 +148,7 @@ HiveSortLimit(fetch=[100]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($2), IS NOT NULL($3))]) HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) HiveProject(sr_item_sk=[$2], sr_ticket_number=[$9]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($9), IS NOT NULL($2))]) HiveTableScan(table=[[default, store_returns]], table:alias=[store_returns]) HiveProject(ws_item_sk=[$0], ws_bill_customer_sk=[$1], $f2=[$2], $f3=[$3], $f4=[$4]) HiveFilter(condition=[>($2, 0)]) @@ -164,7 +164,7 @@ HiveSortLimit(fetch=[100]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($3), IS NOT NULL($4))]) HiveTableScan(table=[[default, web_sales]], table:alias=[web_sales]) HiveProject(wr_item_sk=[$2], wr_order_number=[$13]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($13), IS NOT NULL($2))]) HiveTableScan(table=[[default, web_returns]], table:alias=[web_returns]) HiveProject($f2=[$1], $f3=[$2], IS NOT NULL=[IS NOT NULL($2)], CASE=[CASE(IS NOT NULL($2), $2, 0)], CASE7=[CASE(IS NOT NULL($3), $3, 0)], CASE8=[CASE(IS NOT NULL($4), $4, 0)]) HiveFilter(condition=[>($2, 0)]) @@ -181,6 +181,6 @@ HiveSortLimit(fetch=[100]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($15), IS NOT NULL($3))]) HiveTableScan(table=[[default, catalog_sales]], table:alias=[catalog_sales]) HiveProject(cr_item_sk=[$2], cr_order_number=[$16]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($16), IS NOT NULL($2))]) HiveTableScan(table=[[default, catalog_returns]], table:alias=[catalog_returns]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query80.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query80.q.out index 5d6948cdfa..5387d714b6 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query80.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query80.q.out @@ -238,7 +238,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($2), IS NOT NULL($8))]) HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) HiveProject(sr_item_sk=[$2], sr_ticket_number=[$9], sr_return_amt=[$11], sr_net_loss=[$19]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($9))]) HiveTableScan(table=[[default, store_returns]], table:alias=[store_returns]) HiveProject(d_date_sk=[$0]) HiveFilter(condition=[AND(BETWEEN(false, CAST($2):TIMESTAMP(9), 1998-08-04 00:00:00:TIMESTAMP(9), 1998-09-03 00:00:00:TIMESTAMP(9)), IS NOT NULL($0))]) @@ -264,7 +264,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($12), IS NOT NULL($15), IS NOT NULL($16))]) HiveTableScan(table=[[default, catalog_sales]], table:alias=[catalog_sales]) HiveProject(cr_item_sk=[$2], cr_order_number=[$16], cr_return_amount=[$18], cr_net_loss=[$26]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($16))]) HiveTableScan(table=[[default, catalog_returns]], table:alias=[catalog_returns]) HiveProject(d_date_sk=[$0]) HiveFilter(condition=[AND(BETWEEN(false, CAST($2):TIMESTAMP(9), 1998-08-04 00:00:00:TIMESTAMP(9), 1998-09-03 00:00:00:TIMESTAMP(9)), IS NOT NULL($0))]) @@ -290,7 +290,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($13), IS NOT NULL($3), IS NOT NULL($16))]) HiveTableScan(table=[[default, web_sales]], table:alias=[web_sales]) HiveProject(wr_item_sk=[$2], wr_order_number=[$13], wr_return_amt=[$15], wr_net_loss=[$23]) - HiveFilter(condition=[IS NOT NULL($2)]) + HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($13))]) HiveTableScan(table=[[default, web_returns]], table:alias=[web_returns]) HiveProject(d_date_sk=[$0]) HiveFilter(condition=[AND(BETWEEN(false, CAST($2):TIMESTAMP(9), 1998-08-04 00:00:00:TIMESTAMP(9), 1998-09-03 00:00:00:TIMESTAMP(9)), IS NOT NULL($0))]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query93.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query93.q.out index b0b8169a91..ee9f9789d8 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query93.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query93.q.out @@ -51,7 +51,7 @@ HiveSortLimit(sort0=[$1], sort1=[$0], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales]) HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(sr_item_sk=[$2], sr_reason_sk=[$8], sr_ticket_number=[$9], sr_return_quantity=[$10]) - HiveFilter(condition=[AND(IS NOT NULL($8), IS NOT NULL($2), IS NOT NULL($9))]) + HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($9), IS NOT NULL($8))]) HiveTableScan(table=[[default, store_returns]], table:alias=[store_returns]) HiveProject(r_reason_sk=[$0]) HiveFilter(condition=[AND(=($2, _UTF-16LE'Did not like the warranty'), IS NOT NULL($0))]) diff --git a/ql/src/test/results/clientpositive/perf/tez/query40.q.out b/ql/src/test/results/clientpositive/perf/tez/query40.q.out index 9d64330048..fd5f97d133 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query40.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query40.q.out @@ -169,7 +169,7 @@ Stage-0 Select Operator [SEL_114] (rows=28798881 width=117) Output:["_col0","_col1","_col2"] Filter Operator [FIL_113] (rows=28798881 width=117) - predicate:cr_item_sk is not null + predicate:(cr_item_sk is not null and cr_order_number is not null) TableScan [TS_3] (rows=28798881 width=117) default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:COMPLETE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash"] diff --git a/ql/src/test/results/clientpositive/perf/tez/query45.q.out b/ql/src/test/results/clientpositive/perf/tez/query45.q.out index 19d74d98b7..6a14e4fcac 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query45.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query45.q.out @@ -179,7 +179,7 @@ Stage-0 Select Operator [SEL_143] (rows=11 width=104) Output:["i_item_id"] Filter Operator [FIL_141] (rows=11 width=104) - predicate:(i_item_sk) IN (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) + predicate:((i_item_sk) IN (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) and i_item_id is not null) Please refer to the previous TableScan [TS_6] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_43] diff --git a/ql/src/test/results/clientpositive/perf/tez/query72.q.out b/ql/src/test/results/clientpositive/perf/tez/query72.q.out index f6a99a35de..b3b4035cac 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query72.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query72.q.out @@ -101,182 +101,184 @@ Stage-0 limit:100 Stage-1 Reducer 7 vectorized - File Output Operator [FS_293] - Limit [LIM_292] (rows=100 width=312) + File Output Operator [FS_295] + Limit [LIM_294] (rows=100 width=312) Number of rows:100 - Select Operator [SEL_291] (rows=384313734 width=312) + Select Operator [SEL_293] (rows=384313734 width=312) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Reducer 6 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_290] - Group By Operator [GBY_289] (rows=384313734 width=312) + SHUFFLE [RS_292] + Group By Operator [GBY_291] (rows=384313734 width=312) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2 <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_69] + SHUFFLE [RS_70] PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_68] (rows=610435044 width=312) + Group By Operator [GBY_69] (rows=610435044 width=312) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col3)","count(_col4)","count()"],keys:_col0, _col1, _col2 - Select Operator [SEL_66] (rows=1574305390 width=292) + Select Operator [SEL_67] (rows=1574305390 width=292) Output:["_col0","_col1","_col2","_col3","_col4"] - Merge Join Operator [MERGEJOIN_251] (rows=1574305390 width=292) - Conds:RS_63._col4, _col6=RS_288._col0, _col1(Left Outer),Output:["_col13","_col15","_col19","_col25"] + Merge Join Operator [MERGEJOIN_252] (rows=1574305390 width=292) + Conds:RS_64._col4, _col6=RS_290._col0, _col1(Left Outer),Output:["_col13","_col15","_col19","_col25"] <-Map 24 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_288] + SHUFFLE [RS_290] PartitionCols:_col0, _col1 - Select Operator [SEL_287] (rows=28798881 width=8) + Select Operator [SEL_289] (rows=28798881 width=8) Output:["_col0","_col1"] - Filter Operator [FIL_286] (rows=28798881 width=8) - predicate:cr_item_sk is not null - TableScan [TS_60] (rows=28798881 width=8) + Filter Operator [FIL_288] (rows=28798881 width=8) + predicate:(cr_item_sk is not null and cr_order_number is not null) + TableScan [TS_61] (rows=28798881 width=8) default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:COMPLETE,Output:["cr_item_sk","cr_order_number"] <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_63] + SHUFFLE [RS_64] PartitionCols:_col4, _col6 - Select Operator [SEL_59] (rows=610435044 width=300) + Select Operator [SEL_60] (rows=610435044 width=300) Output:["_col4","_col6","_col13","_col15","_col19","_col25"] - Merge Join Operator [MERGEJOIN_250] (rows=610435044 width=300) - Conds:RS_56._col0, _col19=RS_285._col0, _col1(Inner),Output:["_col5","_col9","_col14","_col16","_col19","_col23"] + Merge Join Operator [MERGEJOIN_251] (rows=610435044 width=300) + Conds:RS_57._col0, _col19=RS_287._col0, _col1(Inner),Output:["_col5","_col9","_col14","_col16","_col19","_col23"] <-Map 23 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_285] + SHUFFLE [RS_287] PartitionCols:_col0, _col1 - Select Operator [SEL_284] (rows=73049 width=8) + Select Operator [SEL_286] (rows=73049 width=8) Output:["_col0","_col1"] - Filter Operator [FIL_283] (rows=73049 width=8) + Filter Operator [FIL_285] (rows=73049 width=8) predicate:(d_date_sk is not null and d_week_seq is not null) - TableScan [TS_46] (rows=73049 width=8) + TableScan [TS_47] (rows=73049 width=8) default@date_dim,d2,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_week_seq"] <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_56] + SHUFFLE [RS_57] PartitionCols:_col0, _col19 - Filter Operator [FIL_55] (rows=545947820 width=311) + Filter Operator [FIL_56] (rows=545947820 width=311) predicate:(_col3 < _col17) - Merge Join Operator [MERGEJOIN_249] (rows=1637843460 width=311) - Conds:RS_52._col1=RS_53._col8(Inner),Output:["_col0","_col3","_col5","_col9","_col14","_col16","_col17","_col19","_col23"] + Merge Join Operator [MERGEJOIN_250] (rows=1637843460 width=311) + Conds:RS_53._col1=RS_54._col8(Inner),Output:["_col0","_col3","_col5","_col9","_col14","_col16","_col17","_col19","_col23"] <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_53] + SHUFFLE [RS_54] PartitionCols:_col8 - Select Operator [SEL_45] (rows=2726340 width=219) + Select Operator [SEL_46] (rows=2726340 width=219) Output:["_col3","_col8","_col10","_col11","_col13","_col17"] - Filter Operator [FIL_44] (rows=2726340 width=219) + Filter Operator [FIL_45] (rows=2726340 width=219) predicate:(_col17 > _col10) - Merge Join Operator [MERGEJOIN_248] (rows=8179022 width=219) - Conds:RS_41._col1=RS_282._col0(Inner),Output:["_col4","_col6","_col7","_col9","_col10","_col13","_col15","_col17"] + Merge Join Operator [MERGEJOIN_249] (rows=8179022 width=219) + Conds:RS_42._col1=RS_284._col0(Inner),Output:["_col4","_col6","_col7","_col9","_col10","_col13","_col15","_col17"] <-Map 22 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_282] + SHUFFLE [RS_284] PartitionCols:_col0 - Select Operator [SEL_281] (rows=73049 width=12) + Select Operator [SEL_283] (rows=73049 width=12) Output:["_col0","_col1"] - Filter Operator [FIL_280] (rows=73049 width=98) + Filter Operator [FIL_282] (rows=73049 width=98) predicate:d_date_sk is not null - TableScan [TS_23] (rows=73049 width=98) + TableScan [TS_24] (rows=73049 width=98) default@date_dim,d3,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_date"] <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_41] + SHUFFLE [RS_42] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_247] (rows=8179022 width=214) - Conds:RS_38._col4=RS_279._col0(Inner),Output:["_col1","_col4","_col6","_col7","_col9","_col10","_col13","_col15"] + Merge Join Operator [MERGEJOIN_248] (rows=8179022 width=214) + Conds:RS_39._col4=RS_281._col0(Inner),Output:["_col1","_col4","_col6","_col7","_col9","_col10","_col13","_col15"] <-Map 21 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_279] + SHUFFLE [RS_281] PartitionCols:_col0 - Select Operator [SEL_278] (rows=462000 width=188) + Select Operator [SEL_280] (rows=462000 width=188) Output:["_col0","_col1"] - Filter Operator [FIL_277] (rows=462000 width=188) + Filter Operator [FIL_279] (rows=462000 width=188) predicate:i_item_sk is not null - TableScan [TS_20] (rows=462000 width=188) + TableScan [TS_21] (rows=462000 width=188) default@item,item,Tbl:COMPLETE,Col:COMPLETE,Output:["i_item_sk","i_item_desc"] <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_38] + SHUFFLE [RS_39] PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_246] (rows=8179022 width=30) - Conds:RS_35._col5=RS_276._col0(Left Outer),Output:["_col1","_col4","_col6","_col7","_col9","_col10","_col13"] + Merge Join Operator [MERGEJOIN_247] (rows=8179022 width=30) + Conds:RS_36._col5=RS_278._col0(Left Outer),Output:["_col1","_col4","_col6","_col7","_col9","_col10","_col13"] <-Map 20 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_276] + SHUFFLE [RS_278] PartitionCols:_col0 - Select Operator [SEL_275] (rows=2300 width=4) + Select Operator [SEL_277] (rows=2300 width=4) Output:["_col0"] - TableScan [TS_18] (rows=2300 width=4) - default@promotion,promotion,Tbl:COMPLETE,Col:COMPLETE,Output:["p_promo_sk"] + Filter Operator [FIL_276] (rows=2300 width=4) + predicate:p_promo_sk is not null + TableScan [TS_18] (rows=2300 width=4) + default@promotion,promotion,Tbl:COMPLETE,Col:COMPLETE,Output:["p_promo_sk"] <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_35] + SHUFFLE [RS_36] PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_245] (rows=8179022 width=29) - Conds:RS_32._col3=RS_274._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7","_col9","_col10"] + Merge Join Operator [MERGEJOIN_246] (rows=8179022 width=29) + Conds:RS_33._col3=RS_275._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7","_col9","_col10"] <-Map 19 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_274] + SHUFFLE [RS_275] PartitionCols:_col0 - Select Operator [SEL_273] (rows=1440 width=4) + Select Operator [SEL_274] (rows=1440 width=4) Output:["_col0"] - Filter Operator [FIL_272] (rows=1440 width=96) + Filter Operator [FIL_273] (rows=1440 width=96) predicate:((hd_buy_potential = '1001-5000') and hd_demo_sk is not null) TableScan [TS_15] (rows=7200 width=96) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:COMPLETE,Output:["hd_demo_sk","hd_buy_potential"] <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_32] + SHUFFLE [RS_33] PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_244] (rows=40895108 width=35) - Conds:RS_29._col2=RS_271._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7","_col9","_col10"] + Merge Join Operator [MERGEJOIN_245] (rows=40895108 width=35) + Conds:RS_30._col2=RS_272._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7","_col9","_col10"] <-Map 18 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_271] + SHUFFLE [RS_272] PartitionCols:_col0 - Select Operator [SEL_270] (rows=265971 width=4) + Select Operator [SEL_271] (rows=265971 width=4) Output:["_col0"] - Filter Operator [FIL_269] (rows=265971 width=89) + Filter Operator [FIL_270] (rows=265971 width=89) predicate:((cd_marital_status = 'M') and cd_demo_sk is not null) TableScan [TS_12] (rows=1861800 width=89) default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:COMPLETE,Output:["cd_demo_sk","cd_marital_status"] <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_29] + SHUFFLE [RS_30] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_243] (rows=100076475 width=39) - Conds:RS_268._col0=RS_260._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10"] + Merge Join Operator [MERGEJOIN_244] (rows=100076475 width=39) + Conds:RS_269._col0=RS_261._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10"] <-Map 16 [SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_260] + PARTITION_ONLY_SHUFFLE [RS_261] PartitionCols:_col0 - Select Operator [SEL_259] (rows=652 width=16) + Select Operator [SEL_260] (rows=652 width=16) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_258] (rows=652 width=106) + Filter Operator [FIL_259] (rows=652 width=106) predicate:((d_year = 2001) and d_date_sk is not null and d_week_seq is not null) TableScan [TS_9] (rows=73049 width=106) default@date_dim,d1,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_date","d_week_seq","d_year"] <-Map 9 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_268] + SHUFFLE [RS_269] PartitionCols:_col0 - Select Operator [SEL_267] (rows=282274763 width=31) + Select Operator [SEL_268] (rows=282274763 width=31) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_266] (rows=282274763 width=31) - predicate:((cs_sold_date_sk BETWEEN DynamicValue(RS_27_d1_d_date_sk_min) AND DynamicValue(RS_27_d1_d_date_sk_max) and in_bloom_filter(cs_sold_date_sk, DynamicValue(RS_27_d1_d_date_sk_bloom_filter))) and cs_bill_cdemo_sk is not null and cs_bill_hdemo_sk is not null and cs_item_sk is not null and cs_ship_date_sk is not null and cs_sold_date_sk is not null) + Filter Operator [FIL_267] (rows=282274763 width=31) + predicate:((cs_sold_date_sk BETWEEN DynamicValue(RS_28_d1_d_date_sk_min) AND DynamicValue(RS_28_d1_d_date_sk_max) and in_bloom_filter(cs_sold_date_sk, DynamicValue(RS_28_d1_d_date_sk_bloom_filter))) and cs_bill_cdemo_sk is not null and cs_bill_hdemo_sk is not null and cs_item_sk is not null and cs_ship_date_sk is not null and cs_sold_date_sk is not null) TableScan [TS_6] (rows=287989836 width=31) default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["cs_sold_date_sk","cs_ship_date_sk","cs_bill_cdemo_sk","cs_bill_hdemo_sk","cs_item_sk","cs_promo_sk","cs_order_number","cs_quantity"] <-Reducer 17 [BROADCAST_EDGE] vectorized - BROADCAST [RS_265] - Group By Operator [GBY_264] (rows=1 width=12) + BROADCAST [RS_266] + Group By Operator [GBY_265] (rows=1 width=12) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"] <-Map 16 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_263] - Group By Operator [GBY_262] (rows=1 width=12) + PARTITION_ONLY_SHUFFLE [RS_264] + Group By Operator [GBY_263] (rows=1 width=12) Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"] - Select Operator [SEL_261] (rows=652 width=4) + Select Operator [SEL_262] (rows=652 width=4) Output:["_col0"] - Please refer to the previous Select Operator [SEL_259] + Please refer to the previous Select Operator [SEL_260] <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_52] + SHUFFLE [RS_53] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_242] (rows=37584000 width=111) - Conds:RS_254._col2=RS_257._col0(Inner),Output:["_col0","_col1","_col3","_col5"] + Merge Join Operator [MERGEJOIN_243] (rows=37584000 width=111) + Conds:RS_255._col2=RS_258._col0(Inner),Output:["_col0","_col1","_col3","_col5"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_254] + SHUFFLE [RS_255] PartitionCols:_col2 - Select Operator [SEL_253] (rows=37584000 width=15) + Select Operator [SEL_254] (rows=37584000 width=15) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_252] (rows=37584000 width=15) + Filter Operator [FIL_253] (rows=37584000 width=15) predicate:(inv_date_sk is not null and inv_item_sk is not null and inv_warehouse_sk is not null) TableScan [TS_0] (rows=37584000 width=15) default@inventory,inventory,Tbl:COMPLETE,Col:COMPLETE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] <-Map 8 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_257] + SHUFFLE [RS_258] PartitionCols:_col0 - Select Operator [SEL_256] (rows=27 width=104) + Select Operator [SEL_257] (rows=27 width=104) Output:["_col0","_col1"] - Filter Operator [FIL_255] (rows=27 width=104) + Filter Operator [FIL_256] (rows=27 width=104) predicate:w_warehouse_sk is not null TableScan [TS_3] (rows=27 width=104) default@warehouse,warehouse,Tbl:COMPLETE,Col:COMPLETE,Output:["w_warehouse_sk","w_warehouse_name"] diff --git a/ql/src/test/results/clientpositive/perf/tez/query75.q.out b/ql/src/test/results/clientpositive/perf/tez/query75.q.out index 4019788fa1..ccde4c59a2 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query75.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query75.q.out @@ -245,7 +245,7 @@ Stage-0 Select Operator [SEL_603] (rows=28798881 width=121) Output:["_col0","_col1","_col2","_col3"] Filter Operator [FIL_602] (rows=28798881 width=121) - predicate:cr_item_sk is not null + predicate:(cr_item_sk is not null and cr_order_number is not null) TableScan [TS_9] (rows=28798881 width=121) default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:COMPLETE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] <-Reducer 22 [SIMPLE_EDGE] @@ -311,7 +311,7 @@ Stage-0 Select Operator [SEL_625] (rows=57591150 width=119) Output:["_col0","_col1","_col2","_col3"] Filter Operator [FIL_624] (rows=57591150 width=119) - predicate:sr_item_sk is not null + predicate:(sr_item_sk is not null and sr_ticket_number is not null) TableScan [TS_31] (rows=57591150 width=119) default@store_returns,store_returns,Tbl:COMPLETE,Col:COMPLETE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] <-Reducer 30 [SIMPLE_EDGE] @@ -371,7 +371,7 @@ Stage-0 Select Operator [SEL_634] (rows=14398467 width=118) Output:["_col0","_col1","_col2","_col3"] Filter Operator [FIL_633] (rows=14398467 width=118) - predicate:wr_item_sk is not null + predicate:(wr_item_sk is not null and wr_order_number is not null) TableScan [TS_60] (rows=14398467 width=118) default@web_returns,web_returns,Tbl:COMPLETE,Col:COMPLETE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] <-Reducer 34 [SIMPLE_EDGE] diff --git a/ql/src/test/results/clientpositive/perf/tez/query78.q.out b/ql/src/test/results/clientpositive/perf/tez/query78.q.out index ba1d91ebcd..e66d6f518a 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query78.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query78.q.out @@ -228,7 +228,7 @@ Stage-0 Select Operator [SEL_267] (rows=28798881 width=8) Output:["_col0","_col1"] Filter Operator [FIL_266] (rows=28798881 width=8) - predicate:cr_item_sk is not null + predicate:(cr_item_sk is not null and cr_order_number is not null) TableScan [TS_51] (rows=28798881 width=8) default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:COMPLETE,Output:["cr_item_sk","cr_order_number"] <-Reducer 4 [SIMPLE_EDGE] @@ -289,7 +289,7 @@ Stage-0 Select Operator [SEL_244] (rows=57591150 width=8) Output:["_col0","_col1"] Filter Operator [FIL_243] (rows=57591150 width=8) - predicate:sr_item_sk is not null + predicate:(sr_item_sk is not null and sr_ticket_number is not null) TableScan [TS_6] (rows=57591150 width=8) default@store_returns,store_returns,Tbl:COMPLETE,Col:COMPLETE,Output:["sr_item_sk","sr_ticket_number"] <-Reducer 9 [ONE_TO_ONE_EDGE] vectorized @@ -347,7 +347,7 @@ Stage-0 Select Operator [SEL_255] (rows=14398467 width=8) Output:["_col0","_col1"] Filter Operator [FIL_254] (rows=14398467 width=8) - predicate:wr_item_sk is not null + predicate:(wr_item_sk is not null and wr_order_number is not null) TableScan [TS_28] (rows=14398467 width=8) default@web_returns,web_returns,Tbl:COMPLETE,Col:COMPLETE,Output:["wr_item_sk","wr_order_number"] diff --git a/ql/src/test/results/clientpositive/perf/tez/query80.q.out b/ql/src/test/results/clientpositive/perf/tez/query80.q.out index 6c4e6ae627..d74ac0db89 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query80.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query80.q.out @@ -363,7 +363,7 @@ Stage-0 Select Operator [SEL_445] (rows=28798881 width=227) Output:["_col0","_col1","_col2","_col3"] Filter Operator [FIL_444] (rows=28798881 width=227) - predicate:cr_item_sk is not null + predicate:(cr_item_sk is not null and cr_order_number is not null) TableScan [TS_42] (rows=28798881 width=227) default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:COMPLETE,Output:["cr_item_sk","cr_order_number","cr_return_amount","cr_net_loss"] <-Reducer 24 [CONTAINS] vectorized @@ -453,7 +453,7 @@ Stage-0 Select Operator [SEL_461] (rows=14398467 width=221) Output:["_col0","_col1","_col2","_col3"] Filter Operator [FIL_460] (rows=14398467 width=221) - predicate:wr_item_sk is not null + predicate:(wr_item_sk is not null and wr_order_number is not null) TableScan [TS_82] (rows=14398467 width=221) default@web_returns,web_returns,Tbl:COMPLETE,Col:COMPLETE,Output:["wr_item_sk","wr_order_number","wr_return_amt","wr_net_loss"] <-Reducer 7 [CONTAINS] vectorized @@ -543,7 +543,7 @@ Stage-0 Select Operator [SEL_413] (rows=57591150 width=224) Output:["_col0","_col1","_col2","_col3"] Filter Operator [FIL_412] (rows=57591150 width=224) - predicate:sr_item_sk is not null + predicate:(sr_item_sk is not null and sr_ticket_number is not null) TableScan [TS_3] (rows=57591150 width=224) default@store_returns,store_returns,Tbl:COMPLETE,Col:COMPLETE,Output:["sr_item_sk","sr_ticket_number","sr_return_amt","sr_net_loss"] diff --git a/ql/src/test/results/clientpositive/position_alias_test_1.q.out b/ql/src/test/results/clientpositive/position_alias_test_1.q.out index 397daf3e29..f56e5bd571 100644 --- a/ql/src/test/results/clientpositive/position_alias_test_1.q.out +++ b/ql/src/test/results/clientpositive/position_alias_test_1.q.out @@ -22,7 +22,7 @@ POSTHOOK: query: create table alias_test_03(a INT, b STRING) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@alias_test_03 -Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from alias_test_01 alias01 @@ -151,17 +151,21 @@ STAGE PLANS: value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string) TableScan alias: alias03 + filterExpr: a is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: a is not null (type: boolean) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: a (type: int), b (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: diff --git a/ql/src/test/results/clientpositive/skewjoin_mapjoin1.q.out b/ql/src/test/results/clientpositive/skewjoin_mapjoin1.q.out index a5afa605c1..d16a67a596 100644 --- a/ql/src/test/results/clientpositive/skewjoin_mapjoin1.q.out +++ b/ql/src/test/results/clientpositive/skewjoin_mapjoin1.q.out @@ -210,9 +210,10 @@ STAGE PLANS: $hdt$_0:a TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (((key = '2') or (key = '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -227,7 +228,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: ((not ((key = '2') or (key = '3'))) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -510,9 +511,10 @@ STAGE PLANS: $hdt$_0:a TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (((key = '2') or (key = '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) @@ -527,7 +529,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: ((not ((key = '2') or (key = '3'))) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) diff --git a/ql/src/test/results/clientpositive/skewjoin_mapjoin10.q.out b/ql/src/test/results/clientpositive/skewjoin_mapjoin10.q.out index 1ff7f0e3fd..bca34babe1 100644 --- a/ql/src/test/results/clientpositive/skewjoin_mapjoin10.q.out +++ b/ql/src/test/results/clientpositive/skewjoin_mapjoin10.q.out @@ -242,9 +242,10 @@ STAGE PLANS: $hdt$_0:a TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 6 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((key = 2) or (key = 3)) (type: boolean) + predicate: (((key = 2) or (key = 3)) and key is not null) (type: boolean) Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), val (type: string) @@ -259,7 +260,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 6 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not ((key = 2) or (key = 3))) (type: boolean) + predicate: ((not ((key = 2) or (key = 3))) and key is not null) (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), val (type: string) @@ -542,9 +543,10 @@ STAGE PLANS: $hdt$_0:a TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((key = 2) or (key = 3)) (type: boolean) + predicate: (((key = 2) or (key = 3)) and key is not null) (type: boolean) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int) @@ -559,7 +561,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not ((key = 2) or (key = 3))) (type: boolean) + predicate: ((not ((key = 2) or (key = 3))) and key is not null) (type: boolean) Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int) diff --git a/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out b/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out index 349e9f33fe..aae0223ae6 100644 --- a/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out +++ b/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out @@ -205,9 +205,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (((key = '2') or (key = '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -259,7 +260,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: ((not ((key = '2') or (key = '3'))) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -533,9 +534,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (((key = '2') or (key = '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -598,7 +600,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: ((not ((key = '2') or (key = '3'))) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) diff --git a/ql/src/test/results/clientpositive/skewjoinopt1.q.out b/ql/src/test/results/clientpositive/skewjoinopt1.q.out index 8389ee70df..4f13d8e744 100644 --- a/ql/src/test/results/clientpositive/skewjoinopt1.q.out +++ b/ql/src/test/results/clientpositive/skewjoinopt1.q.out @@ -229,9 +229,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (((key = '2') or (key = '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -306,7 +307,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: ((not ((key = '2') or (key = '3'))) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -577,9 +578,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (((key = '2') or (key = '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) @@ -670,7 +672,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((key = '2') or (key = '3'))) (type: boolean) + predicate: ((not ((key = '2') or (key = '3'))) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) diff --git a/ql/src/test/results/clientpositive/skewjoinopt2.q.out b/ql/src/test/results/clientpositive/skewjoinopt2.q.out index 4bad303039..fd88f425dd 100644 --- a/ql/src/test/results/clientpositive/skewjoinopt2.q.out +++ b/ql/src/test/results/clientpositive/skewjoinopt2.q.out @@ -237,9 +237,10 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE TableScan alias: b + filterExpr: (key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) (type: boolean) + predicate: (((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) and key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -312,7 +313,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) (type: boolean) + predicate: ((not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) and key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -590,9 +591,10 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE TableScan alias: b + filterExpr: (key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) (type: boolean) + predicate: (((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) and key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -691,7 +693,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) (type: boolean) + predicate: ((not ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8'))) and key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_47.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_47.q.out index 20a7348f64..3acadcb3a6 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin_47.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin_47.q.out @@ -803,7 +803,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test1_n8 POSTHOOK: Input: default@test2_n5 #### A masked pattern was here #### -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test2_n5 @@ -841,7 +841,7 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Sorted Merge Bucket Map Join Operator condition map: - Right Outer Join 0 to 1 + Left Outer Join 0 to 1 keys: 0 _col1 (type: int) 1 _col1 (type: int) @@ -868,9 +868,9 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - residual filter predicates: {((_col6 + _col3) >= 100)} + residual filter predicates: {((_col6 + _col0) >= 100)} Select Operator - expressions: _col3 (type: int), _col4 (type: int), _col5 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string), _col0 (type: int), _col1 (type: int), _col2 (type: string) + expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Limit Number of rows: 10 @@ -887,7 +887,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM test2_n5 JOIN test1_n8 a ON (a.key+test2_n5.key >= 100) @@ -910,13 +910,13 @@ POSTHOOK: Input: default@test2_n5 104 3 Fli 99 0 Alice NULL NULL NULL 102 2 Del 100 1 Bob 99 2 Mat 102 2 Del 99 0 Alice 99 2 Mat -103 2 Ema 100 1 Bob 99 2 Mat -103 2 Ema 99 0 Alice 99 2 Mat 102 2 Del 100 1 Bob 101 2 Car 102 2 Del 99 0 Alice 101 2 Car +103 2 Ema 100 1 Bob 99 2 Mat +103 2 Ema 99 0 Alice 99 2 Mat 103 2 Ema 100 1 Bob 101 2 Car 103 2 Ema 99 0 Alice 101 2 Car -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test2_n5 @@ -997,7 +997,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM test2_n5 LEFT OUTER JOIN test1_n8 a ON (a.value = test2_n5.value) @@ -1016,7 +1016,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test1_n8 POSTHOOK: Input: default@test2_n5 #### A masked pattern was here #### -Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Stage-7:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test2_n5 @@ -1057,15 +1057,19 @@ STAGE PLANS: $hdt$_0:test2_n5 TableScan alias: test2_n5 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 - 1 + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 + 1 Stage: Stage-7 Map Reduce @@ -1085,7 +1089,7 @@ STAGE PLANS: 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 residual filter predicates: {((_col3 + _col0) >= 100)} - Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1146 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false table: @@ -1129,13 +1133,13 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 19 Data size: 5063 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 3915 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 - Statistics: Num rows: 10 Data size: 2679 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2675 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 2679 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2675 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1152,7 +1156,7 @@ STAGE PLANS: key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1146 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) TableScan alias: b @@ -1175,13 +1179,13 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 19 Data size: 5063 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 15 Data size: 3915 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 - Statistics: Num rows: 10 Data size: 2679 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2675 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 2679 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2675 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1193,7 +1197,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Stage-7:MAPRED' is a cross product PREHOOK: query: SELECT * FROM test2_n5 JOIN test1_n8 a ON (a.key+test2_n5.key >= 100) @@ -1222,7 +1226,7 @@ NULL NULL NULL NULL NULL NULL NULL NULL None 102 2 Del 99 2 Mat 101 2 Car 103 2 Ema 99 0 Alice 101 2 Car 102 2 Del 99 0 Alice 101 2 Car -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM test2_n5 @@ -1303,7 +1307,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM test2_n5 RIGHT OUTER JOIN test1_n8 a ON (a.value = test2_n5.value) diff --git a/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out b/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out index 54a26e1057..53c1a316d1 100644 --- a/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out +++ b/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out @@ -826,16 +826,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: dept_n1 + filterExpr: (deptid is not null and deptname is not null) (type: boolean) Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptid (type: int), deptname (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (deptid is not null and deptname is not null) (type: boolean) Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: int) + Select Operator + expressions: deptid (type: int), deptname (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) + Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -976,17 +980,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: emp_n2 + filterExpr: (deptid is not null and lastname is not null) (type: boolean) Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: lastname (type: string), deptid (type: int), locid (type: int) - outputColumnNames: _col0, _col1, _col2 + Filter Operator + predicate: (deptid is not null and lastname is not null) (type: boolean) Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Select Operator + expressions: lastname (type: string), deptid (type: int), locid (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: int) + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: int) Execution mode: vectorized Map 3 Map Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out index f03929b6fd..61cb8cf425 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out @@ -220,18 +220,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: bucket_small_n17 - filterExpr: pri BETWEEN 1L AND 2L (type: boolean) + filterExpr: (pri BETWEEN 1L AND 2L and key is not null) (type: boolean) Statistics: Num rows: 236 Data size: 2244 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: bigint), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 236 Data size: 2244 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) + Select Operator + expressions: key (type: bigint), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 236 Data size: 2244 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 236 Data size: 2244 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Reducer 2 Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark index ab6d42dc5a..7dbc3bc2aa 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark @@ -220,18 +220,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: bucket_small_n17 - filterExpr: pri BETWEEN 1L AND 2L (type: boolean) + filterExpr: (pri BETWEEN 1L AND 2L and key is not null) (type: boolean) Statistics: Num rows: 236 Data size: 2244 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: bigint), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 236 Data size: 2244 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) + Select Operator + expressions: key (type: bigint), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 236 Data size: 2244 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 236 Data size: 2244 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Reducer 2 Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/innerjoin.q.out b/ql/src/test/results/clientpositive/spark/innerjoin.q.out index d481f6e439..0ccee98d84 100644 --- a/ql/src/test/results/clientpositive/spark/innerjoin.q.out +++ b/ql/src/test/results/clientpositive/spark/innerjoin.q.out @@ -1227,17 +1227,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Reducer 2 Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out b/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out index 7075137eaf..592a043a8f 100644 --- a/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out +++ b/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out @@ -1605,7 +1605,8 @@ INNER JOIN ((SELECT `key`, `value` FROM `default`.`src1` WHERE `value` IS NOT NULL) AS `t2` LEFT JOIN (SELECT `key` -FROM `default`.`src`) AS `t3` ON `t2`.`key` = `t3`.`key`) ON `t0`.`value` = `t2`.`value` +FROM `default`.`src` +WHERE `key` IS NOT NULL) AS `t4` ON `t2`.`key` = `t4`.`key`) ON `t0`.`value` = `t2`.`value` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -1701,20 +1702,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: y + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - tag: 1 - auto parallelism: false + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: false Execution mode: vectorized Path -> Alias: #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out index 936cfd0143..155bd248e5 100644 --- a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out +++ b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out @@ -23,10 +23,10 @@ FROM (SELECT `key`, `value`, `value` = 60 AS `=`, `value` = 50 AS `=3` FROM `default`.`a_n4`) AS `t` LEFT JOIN (SELECT `key`, CAST(50 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 50) AS `t1` ON `t`.`key` = `t1`.`key` AND `t`.`=3` +WHERE `value` = 50 AND `key` IS NOT NULL) AS `t1` ON `t`.`key` = `t1`.`key` AND `t`.`=3` LEFT JOIN (SELECT `key`, CAST(60 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 60) AS `t3` ON `t`.`key` = `t3`.`key` AND `t`.`=` +WHERE `value` = 60 AND `key` IS NOT NULL) AS `t3` ON `t`.`key` = `t3`.`key` AND `t`.`=` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -115,12 +115,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - filterExpr: (value = 50) (type: boolean) + filterExpr: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 50) (type: boolean) + predicate: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), 50 (type: int) @@ -193,12 +193,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c - filterExpr: (value = 60) (type: boolean) + filterExpr: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 60) (type: boolean) + predicate: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), 60 (type: int) @@ -352,12 +352,12 @@ POSTHOOK: Input: default@a_n4 OPTIMIZED SQL: SELECT `t0`.`key`, `t0`.`value`, `t1`.`key` AS `key1`, `t1`.`value` AS `value1`, `t3`.`key` AS `key2`, `t3`.`value` AS `value2` FROM (SELECT `key`, CAST(50 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 50) AS `t0` +WHERE `value` = 50 AND `key` IS NOT NULL) AS `t0` RIGHT JOIN (SELECT `key`, `value`, `value` = 60 AS `=`, `value` = 50 AS `=3` FROM `default`.`a_n4`) AS `t1` ON `t0`.`key` = `t1`.`key` AND `t1`.`=3` LEFT JOIN (SELECT `key`, CAST(60 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 60) AS `t3` ON `t1`.`key` = `t3`.`key` AND `t1`.`=` +WHERE `value` = 60 AND `key` IS NOT NULL) AS `t3` ON `t1`.`key` = `t3`.`key` AND `t1`.`=` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -373,12 +373,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a_n4 - filterExpr: (value = 50) (type: boolean) + filterExpr: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 50) (type: boolean) + predicate: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), 50 (type: int) @@ -524,12 +524,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c - filterExpr: (value = 60) (type: boolean) + filterExpr: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 60) (type: boolean) + predicate: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), 60 (type: int) @@ -683,12 +683,12 @@ POSTHOOK: Input: default@a_n4 OPTIMIZED SQL: SELECT `t0`.`key`, `t0`.`value`, `t1`.`key` AS `key1`, `t1`.`value` AS `value1`, `t3`.`key` AS `key2`, `t3`.`value` AS `value2` FROM (SELECT `key`, CAST(50 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 50) AS `t0` +WHERE `value` = 50 AND `key` IS NOT NULL) AS `t0` RIGHT JOIN (SELECT `key`, `value`, `value` = 60 AS `=`, `value` = 50 AS `=3` FROM `default`.`a_n4`) AS `t1` ON `t0`.`key` = `t1`.`key` AND `t1`.`=3` LEFT JOIN (SELECT `key`, CAST(60 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 60) AS `t3` ON `t1`.`key` = `t3`.`key` AND `t1`.`=` +WHERE `value` = 60 AND `key` IS NOT NULL) AS `t3` ON `t1`.`key` = `t3`.`key` AND `t1`.`=` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -704,12 +704,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a_n4 - filterExpr: (value = 50) (type: boolean) + filterExpr: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 50) (type: boolean) + predicate: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), 50 (type: int) @@ -855,12 +855,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c - filterExpr: (value = 60) (type: boolean) + filterExpr: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 60) (type: boolean) + predicate: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), 60 (type: int) @@ -1018,10 +1018,10 @@ FULL JOIN (SELECT `key`, `value`, `value` = 60 AS `=`, `value` = 50 AS `=3` FROM `default`.`a_n4`) AS `t0` ON `t`.`key` = `t0`.`key` AND `t`.`=3` AND `t0`.`=3` LEFT JOIN (SELECT `key`, CAST(60 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 60) AS `t2` ON `t0`.`key` = `t2`.`key` AND `t0`.`=` +WHERE `value` = 60 AND `key` IS NOT NULL) AS `t2` ON `t0`.`key` = `t2`.`key` AND `t0`.`=` LEFT JOIN (SELECT `key`, CAST(40 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 40) AS `t4` ON `t`.`key` = `t4`.`key` AND `t`.`=` +WHERE `value` = 40 AND `key` IS NOT NULL) AS `t4` ON `t`.`key` = `t4`.`key` AND `t`.`=` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -1183,12 +1183,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c - filterExpr: (value = 60) (type: boolean) + filterExpr: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 60) (type: boolean) + predicate: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), 60 (type: int) @@ -1261,12 +1261,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: d - filterExpr: (value = 40) (type: boolean) + filterExpr: ((value = 40) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 40) (type: boolean) + predicate: ((value = 40) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), 40 (type: int) @@ -1417,13 +1417,13 @@ FROM (SELECT `key`, `value`, `value` = 40 AS `=`, `value` = 60 AS `=3`, `value` FROM `default`.`a_n4`) AS `t` LEFT JOIN (SELECT `key`, CAST(50 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 50) AS `t1` ON `t`.`key` = `t1`.`key` AND `t`.`=4` +WHERE `value` = 50 AND `key` IS NOT NULL) AS `t1` ON `t`.`key` = `t1`.`key` AND `t`.`=4` LEFT JOIN (SELECT `key`, CAST(60 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 60) AS `t3` ON `t`.`key` = `t3`.`key` AND `t`.`=3` +WHERE `value` = 60 AND `key` IS NOT NULL) AS `t3` ON `t`.`key` = `t3`.`key` AND `t`.`=3` LEFT JOIN (SELECT `key`, CAST(40 AS INTEGER) AS `value` FROM `default`.`a_n4` -WHERE `value` = 40) AS `t5` ON `t`.`key` = `t5`.`key` AND `t`.`=` +WHERE `value` = 40 AND `key` IS NOT NULL) AS `t5` ON `t`.`key` = `t5`.`key` AND `t`.`=` STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -1512,12 +1512,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - filterExpr: (value = 50) (type: boolean) + filterExpr: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 50) (type: boolean) + predicate: ((value = 50) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), 50 (type: int) @@ -1590,12 +1590,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c - filterExpr: (value = 60) (type: boolean) + filterExpr: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 60) (type: boolean) + predicate: ((value = 60) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), 60 (type: int) @@ -1668,12 +1668,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: d - filterExpr: (value = 40) (type: boolean) + filterExpr: ((value = 40) and key is not null) (type: boolean) Statistics: Num rows: 3 Data size: 18 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: (value = 40) (type: boolean) + predicate: ((value = 40) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), 40 (type: int) diff --git a/ql/src/test/results/clientpositive/spark/join_merging.q.out b/ql/src/test/results/clientpositive/spark/join_merging.q.out index afa8e1e163..5d49ef5142 100644 --- a/ql/src/test/results/clientpositive/spark/join_merging.q.out +++ b/ql/src/test/results/clientpositive/spark/join_merging.q.out @@ -43,26 +43,30 @@ STAGE PLANS: Map Operator Tree: TableScan alias: p2 + filterExpr: p_partkey is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: p_partkey is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int) Execution mode: vectorized Map 5 Map Operator Tree: TableScan alias: p1 - filterExpr: (p_size > 10) (type: boolean) + filterExpr: ((p_size > 10) and p_partkey is not null) (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (p_size > 10) (type: boolean) + predicate: ((p_size > 10) and p_partkey is not null) (type: boolean) Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_partkey (type: int), p_size (type: int) @@ -100,7 +104,7 @@ STAGE PLANS: Reduce Operator Tree: Join Operator condition map: - Right Outer Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col0 (type: int) 1 _col0 (type: int) diff --git a/ql/src/test/results/clientpositive/spark/join_star.q.out b/ql/src/test/results/clientpositive/spark/join_star.q.out index b1656d2151..afe864164d 100644 --- a/ql/src/test/results/clientpositive/spark/join_star.q.out +++ b/ql/src/test/results/clientpositive/spark/join_star.q.out @@ -553,15 +553,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: dim1 + filterExpr: f1 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f1 (type: int), f2 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f1 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col2 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f1 (type: int), f2 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col2 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -569,15 +573,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: dim2 + filterExpr: f3 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f3 (type: int), f4 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f3 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col4 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f3 (type: int), f4 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col4 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -710,15 +718,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: dim1 + filterExpr: f1 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f1 (type: int), f2 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f1 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col2 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f1 (type: int), f2 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col2 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -726,15 +738,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: dim2 + filterExpr: f3 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f3 (type: int), f4 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f3 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col5 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f3 (type: int), f4 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col5 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -742,15 +758,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: dim3 + filterExpr: f5 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f5 (type: int), f6 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f5 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col3 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f5 (type: int), f6 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col3 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -758,15 +778,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: dim4 + filterExpr: f7 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f7 (type: int), f8 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f7 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col9 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f7 (type: int), f8 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col9 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -774,15 +798,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: dim5 + filterExpr: f9 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f9 (type: int), f10 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f9 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col11 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f9 (type: int), f10 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col11 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -790,15 +818,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: dim6 + filterExpr: f11 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f11 (type: int), f12 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f11 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col9 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f11 (type: int), f12 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col9 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -806,15 +838,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: dim7 + filterExpr: f13 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f13 (type: int), f14 (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: f13 is not null (type: boolean) Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col15 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: f13 (type: int), f14 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col15 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work diff --git a/ql/src/test/results/clientpositive/spark/mapjoin1.q.out b/ql/src/test/results/clientpositive/spark/mapjoin1.q.out index a1658a1851..fffab60a18 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin1.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin1.q.out @@ -45,15 +45,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Local Work: Map Reduce Local Work @@ -141,18 +145,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - filter predicates: - 0 - 1 {_col2} - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + filter predicates: + 0 + 1 {_col2} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Local Work: Map Reduce Local Work @@ -245,18 +253,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - filter predicates: - 0 - 1 {_col2} - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + filter predicates: + 0 + 1 {_col2} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Local Work: Map Reduce Local Work @@ -349,15 +361,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Local Work: Map Reduce Local Work @@ -445,18 +461,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - filter predicates: - 0 - 1 {_col2} - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + filter predicates: + 0 + 1 {_col2} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Local Work: Map Reduce Local Work @@ -549,18 +569,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - filter predicates: - 0 - 1 {_col2} - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + filter predicates: + 0 + 1 {_col2} + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) Local Work: Map Reduce Local Work diff --git a/ql/src/test/results/clientpositive/spark/mergejoins_mixed.q.out b/ql/src/test/results/clientpositive/spark/mergejoins_mixed.q.out index 653e7d28f4..c4fe969cbb 100644 --- a/ql/src/test/results/clientpositive/spark/mergejoins_mixed.q.out +++ b/ql/src/test/results/clientpositive/spark/mergejoins_mixed.q.out @@ -341,17 +341,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: d + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -446,17 +450,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 5 Map Operator Tree: @@ -479,7 +487,7 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 - Right Outer Join 1 to 2 + Inner Join 1 to 2 Right Outer Join 0 to 3 keys: 0 _col0 (type: string) @@ -569,17 +577,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 7 Map Operator Tree: @@ -704,10 +716,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -724,17 +736,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 7 Map Operator Tree: @@ -772,7 +788,7 @@ STAGE PLANS: Reduce Operator Tree: Join Operator condition map: - Right Outer Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col3 (type: string) 1 _col0 (type: string) @@ -1026,17 +1042,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 7 Map Operator Tree: @@ -1177,17 +1197,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 7 Map Operator Tree: @@ -1308,10 +1332,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -1344,17 +1368,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: d + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -1459,10 +1487,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - filterExpr: key is not null (type: boolean) + filterExpr: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -1629,33 +1657,41 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 6 Map Operator Tree: TableScan alias: d + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Reducer 2 Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out b/ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out index 89d9958035..42275f8142 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out @@ -217,9 +217,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (((key = '2') or (key = '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -257,7 +258,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key <> '2') and (key <> '3')) (type: boolean) + predicate: (((key <> '2') and (key <> '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -559,9 +560,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (((key = '2') or (key = '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -599,7 +601,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key <> '2') and (key <> '3')) (type: boolean) + predicate: (((key <> '2') and (key <> '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt1.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt1.q.out index 40b22740b9..a41987f5bb 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt1.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt1.q.out @@ -217,9 +217,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (((key = '2') or (key = '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -257,7 +258,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key <> '2') and (key <> '3')) (type: boolean) + predicate: (((key <> '2') and (key <> '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -537,9 +538,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = '2') or (key = '3')) (type: boolean) + predicate: (((key = '2') or (key = '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) @@ -575,7 +577,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key <> '2') and (key <> '3')) (type: boolean) + predicate: (((key <> '2') and (key <> '3')) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt15.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt15.q.out index 2df2b25c8d..3541f00339 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt15.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt15.q.out @@ -249,9 +249,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 2) or (key = 3)) (type: boolean) + predicate: (((key = 2) or (key = 3)) and key is not null) (type: boolean) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), val (type: string) @@ -289,7 +290,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key <> 2) and (key <> 3)) (type: boolean) + predicate: (((key <> 2) and (key <> 3)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), val (type: string) @@ -569,9 +570,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = 2) or (key = 3)) (type: boolean) + predicate: (((key = 2) or (key = 3)) and key is not null) (type: boolean) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) @@ -607,7 +609,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key <> 2) and (key <> 3)) (type: boolean) + predicate: (((key <> 2) and (key <> 3)) and key is not null) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out index b32312d314..acde927ad9 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out @@ -228,9 +228,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: (key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) (type: boolean) + predicate: (((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) and key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -266,7 +267,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key <> '2') and (key <> '7')) and (key <> '3')) and (key <> '8')) (type: boolean) + predicate: (((((key <> '2') and (key <> '7')) and (key <> '3')) and (key <> '8')) and key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -553,9 +554,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: (key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) (type: boolean) + predicate: (((((key = '2') or (key = '7')) or (key = '3')) or (key = '8')) and key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) @@ -591,7 +593,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((key <> '2') and (key <> '7')) and (key <> '3')) and (key <> '8')) (type: boolean) + predicate: (((((key <> '2') and (key <> '7')) and (key <> '3')) and (key <> '8')) and key is not null and val is not null) (type: boolean) Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), val (type: string) diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out index d2f8b39f59..80d9eed8ae 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out @@ -781,18 +781,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: q + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - 3 _col0 (type: int) - 4 _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + 3 _col0 (type: int) + 4 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -800,18 +804,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: r + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - 3 _col0 (type: int) - 4 _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + 3 _col0 (type: int) + 4 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -819,18 +827,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - 3 _col0 (type: int) - 4 _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + 3 _col0 (type: int) + 4 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -838,18 +850,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t + filterExpr: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - 3 _col0 (type: int) - 4 _col0 (type: int) + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + 3 _col0 (type: int) + 4 _col0 (type: int) Execution mode: vectorized Local Work: Map Reduce Local Work diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out index 2eee709ab2..6e6d6159ab 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out @@ -297,7 +297,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: - Left Outer Join 0 to 1 + Inner Join 0 to 1 Right Outer Join 1 to 2 keys: 0 _col0 (type: int) @@ -382,17 +382,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 4 Map Operator Tree: @@ -708,17 +712,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 3 Map Operator Tree: @@ -933,17 +941,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -1013,43 +1025,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized - Map 4 Map Operator Tree: TableScan alias: c @@ -1058,32 +1036,23 @@ STAGE PLANS: expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized - Reducer 2 - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - Right Outer Join 1 to 2 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 4576 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 4576 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Sorted Merge Bucket Map Join Operator + condition map: + Right Outer Join 0 to 1 + Right Outer Join 1 to 2 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 2 Data size: 4576 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 4576 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out index b05c8ab19d..5ef294aa38 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out @@ -297,7 +297,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: - Left Outer Join 0 to 1 + Inner Join 0 to 1 Right Outer Join 1 to 2 keys: 0 _col0 (type: int) @@ -382,17 +382,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 4 Map Operator Tree: @@ -708,17 +712,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 3 Map Operator Tree: @@ -933,17 +941,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c + filterExpr: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -1013,43 +1025,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark - Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2080 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized - Map 3 - Map Operator Tree: - TableScan - alias: b - Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2060 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized - Map 4 Map Operator Tree: TableScan alias: c @@ -1058,32 +1036,23 @@ STAGE PLANS: expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2220 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - Execution mode: vectorized - Reducer 2 - Reduce Operator Tree: - Join Operator - condition map: - Full Outer Join 0 to 1 - Right Outer Join 1 to 2 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - 2 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 4576 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 4576 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Sorted Merge Bucket Map Join Operator + condition map: + Right Outer Join 0 to 1 + Right Outer Join 1 to 2 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + 2 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 2 Data size: 4576 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 4576 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/spark/spark_constprog_dpp.q.out b/ql/src/test/results/clientpositive/spark/spark_constprog_dpp.q.out index c1842b839d..dea5b0eb63 100644 --- a/ql/src/test/results/clientpositive/spark/spark_constprog_dpp.q.out +++ b/ql/src/test/results/clientpositive/spark/spark_constprog_dpp.q.out @@ -134,11 +134,14 @@ STAGE PLANS: Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: _col0 is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out index 76a70972c9..2b677297c4 100644 --- a/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out @@ -3446,6 +3446,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -3562,6 +3563,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -6223,6 +6225,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -6328,6 +6331,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) diff --git a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out index 60033e706d..dbcc04dff7 100644 --- a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out @@ -1041,25 +1041,27 @@ Stage-0 limit:-1 Stage-1 Reducer 2 - File Output Operator [FS_8] - Select Operator [SEL_7] (rows=85 width=7) + File Output Operator [FS_9] + Select Operator [SEL_8] (rows=77 width=7) Output:["_col0","_col1"] - Join Operator [JOIN_6] (rows=85 width=7) + Join Operator [JOIN_7] (rows=77 width=7) Output:["_col1","_col3"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"} <-Map 1 [PARTITION-LEVEL SORT] - PARTITION-LEVEL SORT [RS_4] + PARTITION-LEVEL SORT [RS_5] PartitionCols:_col0 Select Operator [SEL_1] (rows=20 width=84) Output:["_col0","_col1"] TableScan [TS_0] (rows=20 width=84) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] <-Map 3 [PARTITION-LEVEL SORT] - PARTITION-LEVEL SORT [RS_5] + PARTITION-LEVEL SORT [RS_6] PartitionCols:_col0 - Select Operator [SEL_3] (rows=20 width=84) + Select Operator [SEL_4] (rows=18 width=84) Output:["_col0","_col1"] - TableScan [TS_2] (rows=20 width=84) - default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + Filter Operator [FIL_10] (rows=18 width=84) + predicate:key is not null + TableScan [TS_2] (rows=20 width=84) + default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] PREHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 full outer join cbo_t2 on cbo_t1.key=cbo_t2.key PREHOOK: type: QUERY @@ -2676,52 +2678,54 @@ Stage-0 limit:-1 Stage-1 Reducer 4 - File Output Operator [FS_32] - Select Operator [SEL_31] (rows=27 width=125) + File Output Operator [FS_34] + Select Operator [SEL_33] (rows=27 width=125) Output:["_col0","_col1"] <-Reducer 3 [SORT] - SORT [RS_30] - Select Operator [SEL_29] (rows=27 width=125) + SORT [RS_32] + Select Operator [SEL_31] (rows=27 width=125) Output:["_col0","_col1"] - Filter Operator [FIL_28] (rows=27 width=141) + Filter Operator [FIL_30] (rows=27 width=141) predicate:(((_col3 >= _col2) or (_col2 = 0L) or _col5 is not null or _col1 is null) and (_col1 is not null or (_col2 = 0L) or _col5 is not null) and (_col5 is null or (_col2 = 0L))) - Select Operator [SEL_27] (rows=27 width=141) + Select Operator [SEL_29] (rows=27 width=141) Output:["_col0","_col1","_col2","_col3","_col5"] - Join Operator [JOIN_26] (rows=27 width=141) + Join Operator [JOIN_28] (rows=27 width=141) Output:["_col0","_col1","_col3","_col4","_col5"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{} <-Reducer 2 [PARTITION-LEVEL SORT] - PARTITION-LEVEL SORT [RS_24] - Join Operator [JOIN_23] (rows=27 width=125) + PARTITION-LEVEL SORT [RS_26] + Join Operator [JOIN_25] (rows=27 width=125) Output:["_col0","_col1","_col3"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"UDFToDouble(_col1)","1":"_col0"} <-Map 1 [PARTITION-LEVEL SORT] - PARTITION-LEVEL SORT [RS_21] + PARTITION-LEVEL SORT [RS_23] PartitionCols:UDFToDouble(_col1) Select Operator [SEL_1] (rows=26 width=125) Output:["_col0","_col1"] TableScan [TS_0] (rows=26 width=125) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_size"] <-Reducer 6 [PARTITION-LEVEL SORT] - PARTITION-LEVEL SORT [RS_22] + PARTITION-LEVEL SORT [RS_24] PartitionCols:_col0 - Select Operator [SEL_8] (rows=1 width=12) + Select Operator [SEL_10] (rows=1 width=12) Output:["_col0","_col1"] - Group By Operator [GBY_7] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"] - <-Map 5 [GROUP] - GROUP [RS_6] - Group By Operator [GBY_5] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["sum(p_size)","count(p_size)"] - Filter Operator [FIL_33] (rows=8 width=4) - predicate:(p_size < 10) - TableScan [TS_2] (rows=26 width=4) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] + Filter Operator [FIL_9] (rows=1 width=16) + predicate:(UDFToDouble(_col0) is not null and _col1 is not null) + Group By Operator [GBY_7] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"] + <-Map 5 [GROUP] + GROUP [RS_6] + Group By Operator [GBY_5] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["sum(p_size)","count(p_size)"] + Filter Operator [FIL_35] (rows=8 width=4) + predicate:(p_size < 10) + TableScan [TS_2] (rows=26 width=4) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] <-Reducer 8 [PARTITION-LEVEL SORT] - PARTITION-LEVEL SORT [RS_25] - Group By Operator [GBY_19] (rows=1 width=16) + PARTITION-LEVEL SORT [RS_27] + Group By Operator [GBY_21] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["count()","count(_col0)"] - Select Operator [SEL_15] (rows=1 width=16) + Select Operator [SEL_17] (rows=1 width=16) Output:["_col0"] - Group By Operator [GBY_14] (rows=1 width=16) + Group By Operator [GBY_16] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"] <- Please refer to the previous Map 5 [GROUP] @@ -6354,19 +6358,19 @@ Stage-0 limit:-1 Stage-1 Reducer 3 - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=24) + File Output Operator [FS_15] + Group By Operator [GBY_13] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] <-Reducer 2 [GROUP] - GROUP [RS_11] - Group By Operator [GBY_10] (rows=1 width=24) + GROUP [RS_12] + Group By Operator [GBY_11] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] <-Map 1 [GROUP] - GROUP [RS_9] + GROUP [RS_10] PartitionCols:rand() - Select Operator [SEL_7] (rows=1 width=404) + Select Operator [SEL_8] (rows=1 width=404) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_15] (rows=1 width=404) + Map Join Operator [MAPJOIN_17] (rows=1 width=404) Conds:SEL_1._col2=SEL_1._col1(Left Outer),Output:["_col0","_col1","_col3"] <-Select Operator [SEL_1] (rows=1 width=368) Output:["_col0","_col1","_col2"] @@ -6375,12 +6379,14 @@ Stage-0 Map Reduce Local Work Stage-2 Map 4 - keys: [HASHTABLESINK_17] + keys: [HASHTABLESINK_19] 0_col2,1_col1 - Select Operator [SEL_3] (rows=1 width=184) + Select Operator [SEL_4] (rows=1 width=184) Output:["_col0","_col1"] - TableScan [TS_2] (rows=1 width=184) - default@t2_n68,c,Tbl:COMPLETE,Col:NONE,Output:["key"] + Filter Operator [FIL_16] (rows=1 width=184) + predicate:UDFToDouble(key) is not null + TableScan [TS_2] (rows=1 width=184) + default@t2_n68,c,Tbl:COMPLETE,Col:NONE,Output:["key"] Map Reduce Local Work PREHOOK: query: explain FROM T1_n116 a RIGHT OUTER JOIN T2_n68 c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) @@ -6404,33 +6410,35 @@ Stage-0 limit:-1 Stage-1 Reducer 4 - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=24) + File Output Operator [FS_15] + Group By Operator [GBY_13] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] <-Reducer 3 [GROUP] - GROUP [RS_11] - Group By Operator [GBY_10] (rows=1 width=24) + GROUP [RS_12] + Group By Operator [GBY_11] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] <-Map 2 [GROUP] - GROUP [RS_9] + GROUP [RS_10] PartitionCols:rand() - Select Operator [SEL_7] (rows=1 width=404) + Select Operator [SEL_8] (rows=1 width=404) Output:["_col0","_col1","_col2"] - Map Join Operator [MAPJOIN_15] (rows=1 width=404) - Conds:SEL_3._col2=SEL_3._col1(Right Outer),Output:["_col0","_col1","_col3"] - <-Select Operator [SEL_3] (rows=1 width=184) + Map Join Operator [MAPJOIN_17] (rows=1 width=404) + Conds:SEL_4._col2=SEL_4._col1(Right Outer),Output:["_col0","_col1","_col3"] + <-Select Operator [SEL_4] (rows=1 width=184) Output:["_col0","_col1"] - TableScan [TS_2] (rows=1 width=184) + TableScan [TS_3] (rows=1 width=184) default@t2_n68,c,Tbl:COMPLETE,Col:NONE,Output:["key"] Map Reduce Local Work Stage-2 Map 1 - keys: [HASHTABLESINK_17] + keys: [HASHTABLESINK_19] 0_col2,1_col1 - Select Operator [SEL_1] (rows=1 width=368) + Select Operator [SEL_2] (rows=1 width=368) Output:["_col0","_col1","_col2"] - TableScan [TS_0] (rows=1 width=368) - default@t1_n116,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] + Filter Operator [FIL_16] (rows=1 width=368) + predicate:UDFToDouble(key) is not null + TableScan [TS_0] (rows=1 width=368) + default@t1_n116,a,Tbl:COMPLETE,Col:NONE,Output:["key","val"] Map Reduce Local Work PREHOOK: query: explain FROM T1_n116 a FULL OUTER JOIN T2_n68 c ON c.key+1=a.key select /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) @@ -6503,19 +6511,19 @@ Stage-0 limit:-1 Stage-1 Reducer 3 - File Output Operator [FS_14] - Group By Operator [GBY_12] (rows=1 width=16) + File Output Operator [FS_15] + Group By Operator [GBY_13] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] <-Reducer 2 [GROUP] - GROUP [RS_11] - Group By Operator [GBY_10] (rows=1 width=16) + GROUP [RS_12] + Group By Operator [GBY_11] (rows=1 width=16) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"] <-Map 1 [GROUP] - GROUP [RS_9] + GROUP [RS_10] PartitionCols:rand() - Select Operator [SEL_7] (rows=1 width=202) + Select Operator [SEL_8] (rows=1 width=202) Output:["_col0","_col1"] - Map Join Operator [MAPJOIN_15] (rows=1 width=202) + Map Join Operator [MAPJOIN_17] (rows=1 width=202) Conds:SEL_1._col1=SEL_1._col1(Left Outer),Output:["_col0","_col2"] <-Select Operator [SEL_1] (rows=1 width=184) Output:["_col0","_col1"] @@ -6524,11 +6532,13 @@ Stage-0 Map Reduce Local Work Stage-2 Map 4 - keys: [HASHTABLESINK_17] + keys: [HASHTABLESINK_19] 0_col1,1_col1 - Select Operator [SEL_3] (rows=1 width=368) + Select Operator [SEL_4] (rows=1 width=368) Output:["_col0","_col1"] - TableScan [TS_2] (rows=1 width=368) - default@t1_n116,v,Tbl:COMPLETE,Col:NONE,Output:["val","key"] + Filter Operator [FIL_16] (rows=1 width=368) + predicate:UDFToDouble(key) is not null + TableScan [TS_2] (rows=1 width=368) + default@t1_n116,v,Tbl:COMPLETE,Col:NONE,Output:["key","val"] Map Reduce Local Work diff --git a/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out index 80f26e33a8..15d52dbd17 100644 --- a/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out @@ -6243,6 +6243,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -6459,6 +6460,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -11776,6 +11778,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true @@ -11982,6 +11985,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart + filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true diff --git a/ql/src/test/results/clientpositive/spark/subquery_multi.q.out b/ql/src/test/results/clientpositive/spark/subquery_multi.q.out index 2f48b322cb..551077907d 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_multi.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_multi.q.out @@ -254,7 +254,7 @@ POSTHOOK: Input: default@part_null 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join JOIN[24][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product +Warning: Shuffle Join JOIN[25][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -271,7 +271,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 10 <- Map 9 (GROUP, 2) + Reducer 10 <- Map 4 (GROUP, 2) Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2) Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2), Reducer 8 (PARTITION-LEVEL SORT, 2) Reducer 5 <- Map 4 (GROUP, 2) @@ -303,10 +303,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part_null + filterExpr: p_name is not null (type: boolean) Statistics: Num rows: 1 Data size: 32560 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name + Filter Operator + predicate: p_name is not null (type: boolean) Statistics: Num rows: 1 Data size: 32560 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: p_name (type: string) @@ -340,27 +340,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) Execution mode: vectorized - Map 9 - Map Operator Tree: - TableScan - alias: part_null - filterExpr: p_name is not null (type: boolean) - Statistics: Num rows: 1 Data size: 32560 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 1 Data size: 32560 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: p_name (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 32560 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 32560 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized Reducer 10 Execution mode: vectorized Reduce Operator Tree: @@ -470,7 +449,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[24][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product +Warning: Shuffle Join JOIN[25][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_name from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -479,7 +458,7 @@ POSTHOOK: query: select * from part_null where p_name IN (select p_name from par POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null #### A masked pattern was here #### -Warning: Shuffle Join JOIN[24][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product +Warning: Shuffle Join JOIN[25][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -528,10 +507,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part_null + filterExpr: p_type is not null (type: boolean) Statistics: Num rows: 1 Data size: 32560 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type + Filter Operator + predicate: p_type is not null (type: boolean) Statistics: Num rows: 1 Data size: 32560 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: p_type (type: string) @@ -695,7 +674,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[24][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product +Warning: Shuffle Join JOIN[25][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product PREHOOK: query: select * from part_null where p_name IN (select p_name from part_null) AND p_brand NOT IN (select p_type from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -940,7 +919,7 @@ POSTHOOK: query: select * from part_null where p_brand IN (select p_brand from p POSTHOOK: type: QUERY POSTHOOK: Input: default@part_null #### A masked pattern was here #### -Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -990,10 +969,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tempty + filterExpr: CAST( c AS STRING) is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c (type: char(2)) - outputColumnNames: c + Filter Operator + predicate: CAST( c AS STRING) is not null (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator keys: c (type: char(2)) @@ -1147,7 +1126,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select * from part_null where p_name NOT IN (select c from tempty) AND p_brand IN (select p_brand from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part_null @@ -3043,7 +3022,7 @@ POSTHOOK: Input: default@part_null 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join JOIN[37][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Work 'Reducer 7' is a cross product +Warning: Shuffle Join JOIN[39][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Work 'Reducer 7' is a cross product PREHOOK: query: explain select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -3275,16 +3254,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (_col0 / _col1) (type: double), true (type: boolean) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (_col0 is not null and _col1 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) + Select Operator + expressions: (_col0 / _col1) (type: double), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: boolean) Stage: Stage-0 Fetch Operator @@ -3292,7 +3274,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[37][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Work 'Reducer 7' is a cross product +Warning: Shuffle Join JOIN[39][tables = [$hdt$_1, $hdt$_2, $hdt$_3]] in Work 'Reducer 7' is a cross product PREHOOK: query: select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -4048,7 +4030,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tnull POSTHOOK: Lineage: tnull.c SCRIPT [] POSTHOOK: Lineage: tnull.i SCRIPT [] -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where p_partkey = 3 OR p_size NOT IN (select i from tnull) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4093,10 +4075,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tnull + filterExpr: i is not null (type: boolean) Statistics: Num rows: 3 Data size: 14 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: i + Filter Operator + predicate: i is not null (type: boolean) Statistics: Num rows: 3 Data size: 14 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: i (type: int) @@ -4209,7 +4191,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select * from part where p_partkey = 3 OR p_size NOT IN (select i from tnull) PREHOOK: type: QUERY PREHOOK: Input: default@part diff --git a/ql/src/test/results/clientpositive/spark/subquery_notin.q.out b/ql/src/test/results/clientpositive/spark/subquery_notin.q.out index 34cf381354..b0a6da2171 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_notin.q.out @@ -586,7 +586,7 @@ Manufacturer#4 almond azure aquamarine papaya violet 12 Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#5 almond aquamarine dodger light gainsboro 46 Manufacturer#5 almond azure blanched chiffon midnight 23 -Warning: Shuffle Join JOIN[36][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[38][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select p_name, p_size from @@ -748,16 +748,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: (UDFToDouble(_col0) / _col1) (type: double), true (type: boolean) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (UDFToDouble(_col0) is not null and _col1 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) + Select Operator + expressions: (UDFToDouble(_col0) / _col1) (type: double), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: boolean) Reducer 8 Execution mode: vectorized Reduce Operator Tree: @@ -831,7 +834,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[36][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[38][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select p_name, p_size from part where part.p_size not in @@ -1861,7 +1864,7 @@ POSTHOOK: Input: default@part 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_partkey PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2005,17 +2008,20 @@ STAGE PLANS: expressions: _col1 (type: int) outputColumnNames: _col1 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Filter Operator + predicate: _col1 is not null (type: boolean) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Group By Operator + keys: _col1 (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 7 Execution mode: vectorized Reduce Operator Tree: @@ -2064,7 +2070,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_partkey PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2089,7 +2095,7 @@ POSTHOOK: Input: default@part 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2208,17 +2214,20 @@ STAGE PLANS: expressions: _col1 (type: int) outputColumnNames: _col1 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Filter Operator + predicate: _col1 is not null (type: boolean) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Group By Operator + keys: _col1 (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 6 Execution mode: vectorized Reduce Operator Tree: @@ -2279,7 +2288,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select * from part where (p_partkey*p_size) NOT IN (select min(p_partkey) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2554,7 +2563,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### 26 -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2670,20 +2679,27 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: floor(_col1) (type: bigint) - outputColumnNames: _col0 + expressions: _col1 (type: double) + outputColumnNames: _col1 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Filter Operator + predicate: _col1 is not null (type: boolean) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) + Select Operator + expressions: floor(_col1) (type: bigint) + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: bigint) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 6 Execution mode: vectorized Reduce Operator Tree: @@ -2744,7 +2760,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select * from part where floor(p_retailprice) NOT IN (select floor(min(p_retailprice)) from part group by p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3595,7 +3611,7 @@ almond aquamarine sandy cyan gainsboro almond aquamarine yellow dodger mint almond azure aquamarine papaya violet almond azure blanched chiffon midnight -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -3666,10 +3682,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string) @@ -3854,7 +3870,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -4203,7 +4219,7 @@ POSTHOOK: Input: default@src 5 3 70 3 90 3 -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4347,17 +4363,20 @@ STAGE PLANS: expressions: _col1 (type: int) outputColumnNames: _col1 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Filter Operator + predicate: _col1 is not null (type: boolean) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Group By Operator + keys: _col1 (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 7 Execution mode: vectorized Reduce Operator Tree: @@ -4406,7 +4425,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4431,7 +4450,7 @@ POSTHOOK: Input: default@part 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4579,17 +4598,20 @@ STAGE PLANS: expressions: _col1 (type: int) outputColumnNames: _col1 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: int) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Filter Operator + predicate: _col1 is not null (type: boolean) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Group By Operator + keys: _col1 (type: int) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 7 Execution mode: vectorized Reduce Operator Tree: @@ -4638,7 +4660,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select * from part where (p_size-1) NOT IN (select min(p_size) from part group by p_type) order by p_brand, p_partkey limit 4 PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4651,7 +4673,7 @@ POSTHOOK: Input: default@part 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously 65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -4696,43 +4718,51 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part + filterExpr: p_name is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: p_name is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash + Select Operator + expressions: p_name (type: string) outputColumnNames: _col0 - Statistics: Num rows: 52 Data size: 6294 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 52 Data size: 6294 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 52 Data size: 6294 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 6 Map Operator Tree: TableScan alias: part + filterExpr: p_brand is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_brand (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: p_brand is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash + Select Operator + expressions: p_brand (type: string) outputColumnNames: _col0 - Statistics: Num rows: 52 Data size: 6294 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 52 Data size: 6294 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 52 Data size: 6294 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 7 Map Operator Tree: @@ -4853,7 +4883,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select * from src where key NOT IN (select p_name from part UNION ALL select p_brand from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -5409,10 +5439,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e - filterExpr: UDFToDouble((p_size + 100)) is not null (type: boolean) + filterExpr: (UDFToDouble((p_size + 100)) is not null and p_size is not null) (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble((p_size + 100)) is not null (type: boolean) + predicate: (UDFToDouble((p_size + 100)) is not null and p_size is not null) (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: p_size (type: int) @@ -5640,7 +5670,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@t2_n0 POSTHOOK: Lineage: t2_n0.c1 SCRIPT [] -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain SELECT c1 FROM t1_n0 WHERE c1 NOT IN (SELECT c1 FROM t2_n0) PREHOOK: type: QUERY PREHOOK: Input: default@t1_n0 @@ -5684,10 +5714,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t2_n0 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: c1 + Filter Operator + predicate: c1 is not null (type: boolean) Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: c1 (type: int) @@ -5800,7 +5830,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: SELECT c1 FROM t1_n0 WHERE c1 NOT IN (SELECT c1 FROM t2_n0) PREHOOK: type: QUERY PREHOOK: Input: default@t1_n0 @@ -5859,10 +5889,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1_n0 - filterExpr: UDFToDouble(c2) is not null (type: boolean) + filterExpr: (UDFToDouble(c2) is not null and c2 is not null) (type: boolean) Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble(c2) is not null (type: boolean) + predicate: (UDFToDouble(c2) is not null and c2 is not null) (type: boolean) Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: c2 (type: char(100)) @@ -6995,7 +7025,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@t_n0 #### A masked pattern was here #### 7 -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select t_n0.i from t_n0 where t_n0.j NOT IN (select t1_n0.i from t_n0 t1_n0 ) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 @@ -7038,10 +7068,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1_n0 + filterExpr: i is not null (type: boolean) Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: i + Filter Operator + predicate: i is not null (type: boolean) Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: i (type: int) @@ -7154,7 +7184,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select t_n0.i from t_n0 where t_n0.j NOT IN (select t1_n0.i from t_n0 t1_n0 ) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 @@ -7165,7 +7195,7 @@ POSTHOOK: Input: default@t_n0 #### A masked pattern was here #### 1 4 -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select t_n0.i from t_n0 where t_n0.i NOT IN (select t1_n0.i from t_n0 t1_n0 ) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 @@ -7207,10 +7237,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1_n0 + filterExpr: i is not null (type: boolean) Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: i + Filter Operator + predicate: i is not null (type: boolean) Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: i (type: int) @@ -7323,7 +7353,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select t_n0.i from t_n0 where t_n0.i NOT IN (select t1_n0.i from t_n0 t1_n0 ) PREHOOK: type: QUERY PREHOOK: Input: default@t_n0 @@ -7336,7 +7366,7 @@ PREHOOK: query: drop table t1_n0 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table t1_n0 POSTHOOK: type: DROPTABLE -Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 6' is a cross product +Warning: Shuffle Join JOIN[14][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 6' is a cross product PREHOOK: query: explain select * from src b where b.key not in @@ -7393,10 +7423,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: value + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: value (type: string) @@ -7572,7 +7602,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 6' is a cross product +Warning: Shuffle Join JOIN[14][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 6' is a cross product PREHOOK: query: select * from src b where b.key not in diff --git a/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out b/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out index 3d957668c6..ad9144bb42 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out @@ -3269,8 +3269,8 @@ POSTHOOK: Input: default@part 85768 86428 90681 -Warning: Shuffle Join JOIN[22][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[23][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product PREHOOK: query: explain select * from part_null_n0 where p_name NOT LIKE (select min(p_name) from part_null_n0) AND p_brand NOT IN (select p_name from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3301,10 +3301,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part + filterExpr: p_name is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name + Filter Operator + predicate: p_name is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: p_name (type: string) @@ -3482,8 +3482,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[22][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product -Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[23][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 8' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product PREHOOK: query: select * from part_null_n0 where p_name NOT LIKE (select min(p_name) from part_null_n0) AND p_brand NOT IN (select p_name from part) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3519,7 +3519,7 @@ POSTHOOK: Input: default@part_null_n0 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product PREHOOK: query: explain select * from part_null_n0 where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null_n0 pp where part_null_n0.p_type = pp.p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3550,10 +3550,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part + filterExpr: p_name is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name + Filter Operator + predicate: p_name is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: p_name (type: string) @@ -3751,7 +3751,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product PREHOOK: query: select * from part_null_n0 where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null_n0 pp where part_null_n0.p_type = pp.p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -6015,7 +6015,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product PREHOOK: query: explain select * from part where p_size <> (select count(p_size) from part pp where part.p_type <> pp.p_type) PREHOOK: type: QUERY @@ -6074,10 +6074,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part + filterExpr: p_type is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_type (type: string) - outputColumnNames: p_type + Filter Operator + predicate: p_type is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: p_type (type: string) @@ -6183,7 +6183,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product PREHOOK: query: select * from part where p_size <> (select count(p_size) from part pp where part.p_type <> pp.p_type) PREHOOK: type: QUERY @@ -6237,7 +6237,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@t_n11 POSTHOOK: Lineage: t_n11.i SCRIPT [] POSTHOOK: Lineage: t_n11.j SCRIPT [] -Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product PREHOOK: query: explain select * from t_n11 where 0 = (select count(*) from t_n11 tt_n11 where tt_n11.j <> t_n11.i) PREHOOK: type: QUERY PREHOOK: Input: default@t_n11 @@ -6294,10 +6294,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t_n11 + filterExpr: i is not null (type: boolean) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: i + Filter Operator + predicate: i is not null (type: boolean) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: i (type: int) @@ -6403,7 +6403,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product PREHOOK: query: select * from t_n11 where 0 = (select count(*) from t_n11 tt_n11 where tt_n11.j <> t_n11.i) PREHOOK: type: QUERY PREHOOK: Input: default@t_n11 diff --git a/ql/src/test/results/clientpositive/spark/subquery_select.q.out b/ql/src/test/results/clientpositive/spark/subquery_select.q.out index 23f04add9e..a986c03e26 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_select.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_select.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain SELECT p_size, p_size IN ( SELECT MAX(p_size) FROM part) FROM part @@ -104,16 +104,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: _col0 is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: _col0 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: boolean) Reducer 7 Execution mode: vectorized Reduce Operator Tree: @@ -142,7 +145,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: SELECT p_size, p_size IN ( SELECT MAX(p_size) FROM part) FROM part @@ -375,7 +378,7 @@ POSTHOOK: Input: default@part 6 true 6 true 7 true -Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain SELECT p_size, p_size NOT IN ( SELECT MAX(p_size) FROM part) FROM part @@ -481,16 +484,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: _col0 is not null (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: _col0 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: boolean) Reducer 7 Execution mode: vectorized Reduce Operator Tree: @@ -519,7 +525,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[26][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: SELECT p_size, p_size NOT IN ( SELECT MAX(p_size) FROM part) FROM part @@ -2165,7 +2171,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tnull_n3 POSTHOOK: Lineage: tnull_n3.i EXPRESSION [] -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select p_size, p_size IN (select i from tnull_n3) from part PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2210,10 +2216,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tnull_n3 + filterExpr: i is not null (type: boolean) Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: i + Filter Operator + predicate: i is not null (type: boolean) Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: i (type: int) @@ -2323,7 +2329,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select p_size, p_size IN (select i from tnull_n3) from part PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3127,7 +3133,7 @@ STANDARD ANODIZED TIN true STANDARD BURNISHED TIN true STANDARD PLATED TIN true STANDARD POLISHED STEEL true -Warning: Shuffle Join JOIN[50][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Work 'Reducer 5' is a cross product +Warning: Shuffle Join JOIN[52][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Work 'Reducer 5' is a cross product PREHOOK: query: EXPLAIN SELECT p_size, p_size IN ( SELECT MAX(p_size) FROM part p where p.p_type = part.p_type) AND p_name IN (SELECT min(p_name) from part) @@ -3227,16 +3233,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: _col0 is not null (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: boolean) Reducer 13 Execution mode: vectorized Reduce Operator Tree: @@ -3379,7 +3388,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[50][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Work 'Reducer 5' is a cross product +Warning: Shuffle Join JOIN[52][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Work 'Reducer 5' is a cross product PREHOOK: query: SELECT p_size, p_size IN ( SELECT MAX(p_size) FROM part p where p.p_type = part.p_type) AND p_name IN (SELECT min(p_name) from part) @@ -3562,8 +3571,8 @@ POSTHOOK: Input: default@part 6 false 7 false Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 5' is a cross product -Warning: Shuffle Join JOIN[28][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 10' is a cross product -Warning: Shuffle Join JOIN[44][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 10' is a cross product +Warning: Shuffle Join JOIN[46][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select p_size, (p_size IN (select (select max(p_size) from part) as sb from part order by sb limit 1)) = true from part @@ -3773,16 +3782,19 @@ STAGE PLANS: Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: _col0 is not null (type: boolean) Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: _col0 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: boolean) Reducer 8 Execution mode: vectorized Reduce Operator Tree: @@ -3803,8 +3815,8 @@ STAGE PLANS: ListSink Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 5' is a cross product -Warning: Shuffle Join JOIN[28][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 10' is a cross product -Warning: Shuffle Join JOIN[44][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 10' is a cross product +Warning: Shuffle Join JOIN[46][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select p_size, (p_size IN (select (select max(p_size) from part) as sb from part order by sb limit 1)) = true from part @@ -4250,7 +4262,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### true -Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product PREHOOK: query: explain select o.p_size, (select count(distinct p_type) from part p where p.p_partkey = o.p_partkey) tmp FROM part o right join (select * from part where p_size > (select avg(p_size) from part)) t on t.p_partkey = o.p_partkey PREHOOK: type: QUERY @@ -4279,17 +4291,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: o + filterExpr: p_partkey is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: p_partkey is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: p_partkey (type: int), p_size (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int) Execution mode: vectorized Map 3 Map Operator Tree: @@ -4438,7 +4454,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product PREHOOK: query: select o.p_size, (select count(distinct p_type) from part p where p.p_partkey = o.p_partkey) tmp FROM part o right join (select * from part where p_size > (select avg(p_size) from part)) t on t.p_partkey = o.p_partkey PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/spark/tez_join_tests.q.out b/ql/src/test/results/clientpositive/spark/tez_join_tests.q.out index 2d7b16f281..e577e4b1b3 100644 --- a/ql/src/test/results/clientpositive/spark/tez_join_tests.q.out +++ b/ql/src/test/results/clientpositive/spark/tez_join_tests.q.out @@ -27,32 +27,40 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 5 Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 6 Map Operator Tree: @@ -74,7 +82,7 @@ STAGE PLANS: Reduce Operator Tree: Join Operator condition map: - Right Outer Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) diff --git a/ql/src/test/results/clientpositive/spark/tez_joins_explain.q.out b/ql/src/test/results/clientpositive/spark/tez_joins_explain.q.out index 114e810a54..69feb25194 100644 --- a/ql/src/test/results/clientpositive/spark/tez_joins_explain.q.out +++ b/ql/src/test/results/clientpositive/spark/tez_joins_explain.q.out @@ -27,32 +27,40 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b + filterExpr: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Execution mode: vectorized Map 5 Map Operator Tree: TableScan alias: a + filterExpr: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: key is not null (type: boolean) Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 6 Map Operator Tree: @@ -74,7 +82,7 @@ STAGE PLANS: Reduce Operator Tree: Join Operator condition map: - Right Outer Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col0 (type: string) 1 _col0 (type: string) diff --git a/ql/src/test/results/clientpositive/spark/vector_left_outer_join.q.out b/ql/src/test/results/clientpositive/spark/vector_left_outer_join.q.out index cb50bd6eef..41dc1fd2b2 100644 --- a/ql/src/test/results/clientpositive/spark/vector_left_outer_join.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_left_outer_join.q.out @@ -38,15 +38,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: cint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Map Vectorization: enabled: true @@ -63,15 +67,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hd + filterExpr: ctinyint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 + Filter Operator + predicate: ctinyint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - keys: - 0 _col0 (type: tinyint) - 1 _col0 (type: tinyint) + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 12288 Data size: 2907994 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) Execution mode: vectorized Map Vectorization: enabled: true diff --git a/ql/src/test/results/clientpositive/spark/vector_outer_join0.q.out b/ql/src/test/results/clientpositive/spark/vector_outer_join0.q.out index e6964e1331..4e71cf74b6 100644 --- a/ql/src/test/results/clientpositive/spark/vector_outer_join0.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_outer_join0.q.out @@ -92,25 +92,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t2 + filterExpr: c is not null (type: boolean) Statistics: Num rows: 6 Data size: 550 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:c:int, 1:v2:string, 2:ROW__ID:struct] - Select Operator - expressions: c (type: int), v2 (type: string) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0, 1] + predicateExpression: SelectColumnIsNotNull(col 0:int) + predicate: c is not null (type: boolean) Statistics: Num rows: 6 Data size: 550 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - Spark Hash Table Sink Vectorization: - className: VectorSparkHashTableSinkOperator + Select Operator + expressions: c (type: int), v2 (type: string) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 6 Data size: 550 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + Spark Hash Table Sink Vectorization: + className: VectorSparkHashTableSinkOperator + native: true + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Map Vectorization: enabled: true @@ -252,25 +260,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 + filterExpr: a is not null (type: boolean) Statistics: Num rows: 6 Data size: 544 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:v1:string, 1:a:int, 2:ROW__ID:struct] - Select Operator - expressions: v1 (type: string), a (type: int) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0, 1] + predicateExpression: SelectColumnIsNotNull(col 1:int) + predicate: a is not null (type: boolean) Statistics: Num rows: 6 Data size: 544 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - Spark Hash Table Sink Vectorization: - className: VectorSparkHashTableSinkOperator + Select Operator + expressions: v1 (type: string), a (type: int) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 6 Data size: 544 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + Spark Hash Table Sink Vectorization: + className: VectorSparkHashTableSinkOperator + native: true + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Map Vectorization: enabled: true diff --git a/ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out b/ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out index 71bcdef0d3..510d9ef4aa 100644 --- a/ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out @@ -250,25 +250,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + predicateExpression: SelectColumnIsNotNull(col 2:int) + predicate: cint is not null (type: boolean) Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - Spark Hash Table Sink Vectorization: - className: VectorSparkHashTableSinkOperator + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Select Vectorization: + className: VectorSelectOperator native: true - keys: - 0 _col2 (type: int) - 1 _col2 (type: int) + projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + Spark Hash Table Sink Vectorization: + className: VectorSparkHashTableSinkOperator + native: true + keys: + 0 _col2 (type: int) + 1 _col2 (type: int) Execution mode: vectorized Map Vectorization: enabled: true @@ -431,25 +439,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hd + filterExpr: ctinyint is not null (type: boolean) Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0] + predicateExpression: SelectColumnIsNotNull(col 0:tinyint) + predicate: ctinyint is not null (type: boolean) Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - Spark Hash Table Sink Vectorization: - className: VectorSparkHashTableSinkOperator + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator native: true - keys: - 0 _col0 (type: tinyint) - 1 _col0 (type: tinyint) + projectedOutputColumnNums: [0] + Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + Spark Hash Table Sink Vectorization: + className: VectorSparkHashTableSinkOperator + native: true + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) Execution mode: vectorized Map Vectorization: enabled: true @@ -702,25 +718,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [2] + predicateExpression: SelectColumnIsNotNull(col 2:int) + predicate: cint is not null (type: boolean) Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - Spark Hash Table Sink Vectorization: - className: VectorSparkHashTableSinkOperator + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator native: true - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + projectedOutputColumnNums: [2] + Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + Spark Hash Table Sink Vectorization: + className: VectorSparkHashTableSinkOperator + native: true + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Map Vectorization: enabled: true @@ -743,25 +767,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hd + filterExpr: ctinyint is not null (type: boolean) Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0] + predicateExpression: SelectColumnIsNotNull(col 0:tinyint) + predicate: ctinyint is not null (type: boolean) Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - Spark Hash Table Sink Vectorization: - className: VectorSparkHashTableSinkOperator + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator native: true - keys: - 0 _col0 (type: tinyint) - 1 _col0 (type: tinyint) + projectedOutputColumnNums: [0] + Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + Spark Hash Table Sink Vectorization: + className: VectorSparkHashTableSinkOperator + native: true + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) Execution mode: vectorized Map Vectorization: enabled: true diff --git a/ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out b/ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out index e09b940a23..56d37b0a41 100644 --- a/ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out @@ -266,25 +266,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 20 Data size: 4531 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [2] + predicateExpression: SelectColumnIsNotNull(col 2:int) + predicate: cint is not null (type: boolean) Statistics: Num rows: 20 Data size: 4531 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - Spark Hash Table Sink Vectorization: - className: VectorSparkHashTableSinkOperator + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator native: true - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + projectedOutputColumnNums: [2] + Statistics: Num rows: 20 Data size: 4531 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + Spark Hash Table Sink Vectorization: + className: VectorSparkHashTableSinkOperator + native: true + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) Execution mode: vectorized Map Vectorization: enabled: true @@ -307,25 +315,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hd + filterExpr: cbigint is not null (type: boolean) Statistics: Num rows: 20 Data size: 4531 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] - Select Operator - expressions: cbigint (type: bigint) - outputColumnNames: _col0 - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [3] + predicateExpression: SelectColumnIsNotNull(col 3:bigint) + predicate: cbigint is not null (type: boolean) Statistics: Num rows: 20 Data size: 4531 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator - Spark Hash Table Sink Vectorization: - className: VectorSparkHashTableSinkOperator + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator native: true - keys: - 0 _col1 (type: bigint) - 1 _col0 (type: bigint) + projectedOutputColumnNums: [3] + Statistics: Num rows: 20 Data size: 4531 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator + Spark Hash Table Sink Vectorization: + className: VectorSparkHashTableSinkOperator + native: true + keys: + 0 _col1 (type: bigint) + 1 _col0 (type: bigint) Execution mode: vectorized Map Vectorization: enabled: true diff --git a/ql/src/test/results/clientpositive/subquery_notexists.q.out b/ql/src/test/results/clientpositive/subquery_notexists.q.out index 99551255ab..d8dc8b5f53 100644 --- a/ql/src/test/results/clientpositive/subquery_notexists.q.out +++ b/ql/src/test/results/clientpositive/subquery_notexists.q.out @@ -782,10 +782,10 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: part - filterExpr: UDFToDouble(p_type) is not null (type: boolean) + filterExpr: (UDFToDouble(p_type) is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: UDFToDouble(p_type) is not null (type: boolean) + predicate: (UDFToDouble(p_type) is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_type (type: string) diff --git a/ql/src/test/results/clientpositive/subquery_notin_having.q.out b/ql/src/test/results/clientpositive/subquery_notin_having.q.out index 84787f4009..3efb260668 100644 --- a/ql/src/test/results/clientpositive/subquery_notin_having.q.out +++ b/ql/src/test/results/clientpositive/subquery_notin_having.q.out @@ -519,7 +519,7 @@ POSTHOOK: Input: default@part #### A masked pattern was here #### Manufacturer#1 1173.15 Manufacturer#2 1690.68 -Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain select b.p_mfgr, min(p_retailprice) from part b @@ -673,10 +673,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a + filterExpr: p_mfgr is not null (type: boolean) Statistics: Num rows: 26 Data size: 2756 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_mfgr (type: string), p_retailprice (type: double) - outputColumnNames: p_mfgr, p_retailprice + Filter Operator + predicate: p_mfgr is not null (type: boolean) Statistics: Num rows: 26 Data size: 2756 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: max(p_retailprice), min(p_retailprice) @@ -792,7 +792,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product PREHOOK: query: select b.p_mfgr, min(p_retailprice) from part b group by b.p_mfgr @@ -854,7 +854,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@t2_n85 POSTHOOK: Lineage: t2_n85.c1 SCRIPT [] -Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain SELECT c1 FROM t1_n145 group by c1 having c1 NOT IN (SELECT c1 FROM t2_n85) PREHOOK: type: QUERY PREHOOK: Input: default@t1_n145 @@ -987,11 +987,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t2_n85 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c1 (type: int) - outputColumnNames: c1 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: c1 (type: int) minReductionHashAggr: 0.99 @@ -1061,7 +1061,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product +Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-3:MAPRED' is a cross product PREHOOK: query: SELECT c1 FROM t1_n145 group by c1 having c1 NOT IN (SELECT c1 FROM t2_n85) PREHOOK: type: QUERY PREHOOK: Input: default@t1_n145 diff --git a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out index e0563eb550..d9fde37013 100644 --- a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out +++ b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out @@ -553,7 +553,7 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 1 Map 3 - Statistics: Num rows: 24737 Data size: 197896 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 18466 Data size: 147728 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() minReductionHashAggr: 0.0 @@ -569,16 +569,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Reducer 2 Execution mode: vectorized @@ -667,7 +671,7 @@ STAGE PLANS: 1 _col0 (type: int) input vertices: 1 Map 3 - Statistics: Num rows: 24737 Data size: 197896 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 18466 Data size: 147728 Basic stats: COMPLETE Column stats: COMPLETE HybridGraceHashJoin: true Group By Operator aggregations: count() @@ -684,16 +688,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized Reducer 2 Execution mode: vectorized diff --git a/ql/src/test/results/clientpositive/vector_coalesce_3.q.out b/ql/src/test/results/clientpositive/vector_coalesce_3.q.out index a31d4d7fcc..0890943733 100644 --- a/ql/src/test/results/clientpositive/vector_coalesce_3.q.out +++ b/ql/src/test/results/clientpositive/vector_coalesce_3.q.out @@ -86,15 +86,19 @@ STAGE PLANS: $hdt$_1:n TableScan alias: n + filterExpr: member is not null (type: boolean) Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: member (type: bigint), CASE WHEN (((attr > 1L) or attr is null)) THEN (attr) ELSE (null) END (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: bigint) - 1 _col0 (type: bigint) + Filter Operator + predicate: member is not null (type: boolean) + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: member (type: bigint), CASE WHEN (((attr > 1L) or attr is null)) THEN (attr) ELSE (null) END (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: bigint) + 1 _col0 (type: bigint) Stage: Stage-3 Map Reduce @@ -127,7 +131,7 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: bigint), _col2 (type: bigint) outputColumnNames: _col0, _col1 @@ -135,13 +139,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1] - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out b/ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out index ca1932c335..37e805466c 100644 --- a/ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out @@ -1,6 +1,6 @@ -Warning: Map Join MAPJOIN[53][bigTable=?] in task 'Stage-7:MAPRED' is a cross product -Warning: Map Join MAPJOIN[43][bigTable=?] in task 'Stage-6:MAPRED' is a cross product -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[55][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[45][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain vectorization expression select * from src @@ -45,16 +45,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src + filterExpr: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Select Operator - expressions: key (type: string) - outputColumnNames: key - Select Vectorization: - className: VectorSelectOperator + Filter Operator + Filter Vectorization: + className: VectorFilterOperator native: true - projectedOutputColumnNums: [0] + predicateExpression: SelectColumnIsNotNull(col 0:string) + predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator Group By Vectorization: @@ -534,9 +534,9 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[53][bigTable=?] in task 'Stage-7:MAPRED' is a cross product -Warning: Map Join MAPJOIN[43][bigTable=?] in task 'Stage-6:MAPRED' is a cross product -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[55][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[45][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: select * from src where not key in @@ -565,9 +565,9 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@orcsrc POSTHOOK: Lineage: orcsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Map Join MAPJOIN[53][bigTable=?] in task 'Stage-7:MAPRED' is a cross product -Warning: Map Join MAPJOIN[43][bigTable=?] in task 'Stage-6:MAPRED' is a cross product -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[55][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[45][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: select * from orcsrc where not key in @@ -584,9 +584,9 @@ order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@orcsrc #### A masked pattern was here #### -Warning: Map Join MAPJOIN[53][bigTable=?] in task 'Stage-7:MAPRED' is a cross product -Warning: Map Join MAPJOIN[43][bigTable=?] in task 'Stage-6:MAPRED' is a cross product -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[55][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[45][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: select * from orcsrc where not key in diff --git a/ql/src/test/results/clientpositive/vector_left_outer_join.q.out b/ql/src/test/results/clientpositive/vector_left_outer_join.q.out index 1aa237f65d..bdcdc879fd 100644 --- a/ql/src/test/results/clientpositive/vector_left_outer_join.q.out +++ b/ql/src/test/results/clientpositive/vector_left_outer_join.q.out @@ -43,27 +43,35 @@ STAGE PLANS: $hdt$_1:cd TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) $hdt$_2:hd TableScan alias: hd + filterExpr: ctinyint is not null (type: boolean) Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Statistics: Num rows: 12288 Data size: 36696 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: tinyint) - 1 _col0 (type: tinyint) + Filter Operator + predicate: ctinyint is not null (type: boolean) + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 9173 Data size: 27396 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) Stage: Stage-3 Map Reduce @@ -82,14 +90,14 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 24737 Data size: 86492 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21560 Data size: 73784 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Outer Join 0 to 1 keys: 0 _col0 (type: tinyint) 1 _col0 (type: tinyint) - Statistics: Num rows: 2338217 Data size: 18705736 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1528346 Data size: 12226768 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() minReductionHashAggr: 0.99 diff --git a/ql/src/test/results/clientpositive/vector_left_outer_join2.q.out b/ql/src/test/results/clientpositive/vector_left_outer_join2.q.out index 568fb2a589..78bb7870a9 100644 --- a/ql/src/test/results/clientpositive/vector_left_outer_join2.q.out +++ b/ql/src/test/results/clientpositive/vector_left_outer_join2.q.out @@ -111,18 +111,22 @@ STAGE PLANS: $hdt$_1:tjoin2 TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 {_col3} - 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + filter predicates: + 0 {_col3} + 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Stage: Stage-3 Map Reduce @@ -144,14 +148,14 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -212,18 +216,22 @@ STAGE PLANS: $hdt$_1:tjoin2 TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 {_col3} - 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + filter predicates: + 0 {_col3} + 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Stage: Stage-3 Map Reduce @@ -245,14 +253,14 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -313,18 +321,22 @@ STAGE PLANS: $hdt$_1:tjoin2 TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 {_col3} - 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + filter predicates: + 0 {_col3} + 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Stage: Stage-3 Map Reduce @@ -361,7 +373,7 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.vectorized.execution.mapjoin.native.enabled IS false, hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 @@ -369,13 +381,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3] - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -452,18 +464,22 @@ STAGE PLANS: $hdt$_1:tjoin2 TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 {_col3} - 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + filter predicates: + 0 {_col3} + 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Stage: Stage-3 Map Reduce @@ -500,7 +516,7 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.vectorized.execution.mapjoin.native.enabled IS false, hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 @@ -508,13 +524,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3] - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -591,18 +607,22 @@ STAGE PLANS: $hdt$_1:tjoin2 TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 {_col3} - 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + filter predicates: + 0 {_col3} + 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Stage: Stage-3 Map Reduce @@ -639,7 +659,7 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 @@ -647,13 +667,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3] - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -730,18 +750,22 @@ STAGE PLANS: $hdt$_1:tjoin2 TableScan alias: tjoin2 + filterExpr: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c1 (type: int), c2 (type: char(2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - filter predicates: - 0 {_col3} - 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int), c2 (type: char(2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 270 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + filter predicates: + 0 {_col3} + 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Stage: Stage-3 Map Reduce @@ -778,7 +802,7 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: char(2)) outputColumnNames: _col0, _col1, _col2, _col3 @@ -786,13 +810,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3] - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 6 Data size: 588 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/vector_outer_join0.q.out b/ql/src/test/results/clientpositive/vector_outer_join0.q.out index dc5889c787..f4dc78e761 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join0.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join0.q.out @@ -94,15 +94,19 @@ STAGE PLANS: $hdt$_1:t2 TableScan alias: t2 + filterExpr: c is not null (type: boolean) Statistics: Num rows: 6 Data size: 554 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c (type: int), v2 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 554 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: c is not null (type: boolean) + Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: int), v2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Stage: Stage-3 Map Reduce @@ -135,13 +139,13 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 12 Data size: 2034 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1484 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 12 Data size: 2034 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1484 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -219,15 +223,19 @@ STAGE PLANS: $hdt$_0:t1 TableScan alias: t1 + filterExpr: a is not null (type: boolean) Statistics: Num rows: 6 Data size: 554 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: v1 (type: string), a (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 554 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: v1 (type: string), a (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Stage: Stage-3 Map Reduce @@ -260,13 +268,13 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 12 Data size: 2042 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 1763 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 12 Data size: 2042 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 1763 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/vector_outer_join1.q.out b/ql/src/test/results/clientpositive/vector_outer_join1.q.out index aaf84bab73..b48ff791a7 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join1.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join1.q.out @@ -252,15 +252,19 @@ STAGE PLANS: $hdt$_1:cd TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 15 Data size: 3745 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - Statistics: Num rows: 15 Data size: 3745 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col2 (type: int) - 1 _col2 (type: int) + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 10 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col2 (type: int) + 1 _col2 (type: int) Stage: Stage-3 Map Reduce @@ -293,13 +297,13 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 - Statistics: Num rows: 28 Data size: 15472 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 33 Data size: 14459 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 28 Data size: 15472 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 33 Data size: 14459 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -398,15 +402,19 @@ STAGE PLANS: $hdt$_1:hd TableScan alias: hd + filterExpr: ctinyint is not null (type: boolean) Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: tinyint) - 1 _col0 (type: tinyint) + Filter Operator + predicate: ctinyint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) Stage: Stage-3 Map Reduce @@ -439,13 +447,13 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0 - Statistics: Num rows: 225 Data size: 884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 150 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 225 Data size: 884 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 150 Data size: 584 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -639,27 +647,35 @@ STAGE PLANS: $hdt$_1:cd TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) $hdt$_2:hd TableScan alias: hd + filterExpr: ctinyint is not null (type: boolean) Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: ctinyint (type: tinyint) - outputColumnNames: _col0 - Statistics: Num rows: 15 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: tinyint) - 1 _col0 (type: tinyint) + Filter Operator + predicate: ctinyint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: tinyint) + 1 _col0 (type: tinyint) Stage: Stage-3 Map Reduce @@ -692,7 +708,7 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0 - Statistics: Num rows: 28 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 22 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -707,7 +723,7 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0 - Statistics: Num rows: 420 Data size: 1664 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 220 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col0) Group By Vectorization: diff --git a/ql/src/test/results/clientpositive/vector_outer_join2.q.out b/ql/src/test/results/clientpositive/vector_outer_join2.q.out index a5567d9e99..6789d09c8e 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join2.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join2.q.out @@ -271,27 +271,35 @@ STAGE PLANS: $hdt$_1:cd TableScan alias: cd + filterExpr: cint is not null (type: boolean) Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cint (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) $hdt$_2:hd TableScan alias: hd + filterExpr: cbigint is not null (type: boolean) Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: cbigint (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: bigint) - 1 _col0 (type: bigint) + Filter Operator + predicate: cbigint is not null (type: boolean) + Statistics: Num rows: 10 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: bigint) + 1 _col0 (type: bigint) Stage: Stage-3 Map Reduce @@ -324,7 +332,7 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col1 - Statistics: Num rows: 50 Data size: 328 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 45 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -339,7 +347,7 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col1 - Statistics: Num rows: 100 Data size: 728 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 90 Data size: 648 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), sum(_col1) Group By Vectorization: diff --git a/ql/src/test/results/clientpositive/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/vector_outer_join3.q.out index 3df003d2d2..57a4f0cf48 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join3.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join3.q.out @@ -248,7 +248,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### -{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t0` ON `t`.`cint` = `t0`.`cint`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t1` ON `t`.`cstring1` = `t1`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","Statistics:":"Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_26"}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_4","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_5","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cstring1"],"database:":"default","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 6]"},"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_28","children":{"Map Join Operator":{"columnExprMap:":{"_col1":"0:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col1"],"Statistics:":"Num rows: 50 Data size: 3690 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_29","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 125 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_30","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_31","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_32"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 6]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_15","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_17"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_33"}}}}}} +{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cstring1"],"database:":"default","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 6]"},"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col1":"0:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col1"],"Statistics:":"Num rows: 45 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 101 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 6]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd @@ -292,7 +292,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### -{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t0` ON `t`.`cstring2` = `t0`.`cstring2`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t1` ON `t`.`cstring1` = `t1`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 1488 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Select Operator":{"expressions:":"cstring2 (type: string)","columnExprMap:":{"_col0":"cstring2"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 1488 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_26"}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_4","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_5","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cstring1","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[6, 7]"},"Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_28","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 7:string"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 26 Data size: 1530 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_29","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 65 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_30","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_31","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_32"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_15","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_17"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_33"}}}}}} +{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cstring2"],"database:":"default","filterExpr:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 1488 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cstring2 (type: string)","columnExprMap:":{"_col0":"cstring2"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cstring1","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[6, 7]"},"Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 7:string"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 23 Data size: 1260 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 51 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd @@ -336,7 +336,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### -{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cbigint`, `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cbigint`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t0` ON `t`.`cstring2` = `t0`.`cstring2` AND `t`.`cbigint` = `t0`.`cbigint`\nLEFT JOIN (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t1` ON `t`.`cstring1` = `t1`.`cstring1` AND `t`.`cint` = `t1`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cbigint","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Select Operator":{"expressions:":"cbigint (type: bigint), cstring2 (type: string)","columnExprMap:":{"_col0":"cbigint","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 20 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_26"}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cint","cstring1"],"database:":"default","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_4","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_5","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cbigint","cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cbigint","_col2":"cstring1","_col3":"cstring2"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 3, 6, 7]"},"Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_28","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 3:bigint","col 7:string"],"bigTableValueExpressions:":["col 2:int","col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 26 Data size: 1598 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_29","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:int","col 1:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 65 Data size: 520 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_30","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_31","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_32"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 3, 6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_15","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_17"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_33"}}}}}} +{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cbigint`, `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cbigint`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL AND `cbigint` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2` AND `t`.`cbigint` = `t1`.`cbigint`\nLEFT JOIN (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL AND `cint` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1` AND `t`.`cint` = `t3`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cbigint","cstring2"],"database:":"default","filterExpr:":"(cstring2 is not null and cbigint is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"(cbigint is not null and cstring2 is not null) (type: boolean)","Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cbigint (type: bigint), cstring2 (type: string)","columnExprMap:":{"_col0":"cbigint","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cint","cstring1"],"database:":"default","filterExpr:":"(cstring1 is not null and cint is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"(cint is not null and cstring1 is not null) (type: boolean)","Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cbigint","cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cbigint","_col2":"cstring1","_col3":"cstring2"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 3, 6, 7]"},"Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 3:bigint","col 7:string"],"bigTableValueExpressions:":["col 2:int","col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:int","col 1:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 32 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 3, 6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd diff --git a/ql/src/test/results/clientpositive/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/vector_outer_join4.q.out index 826a838d36..1cfd6f7a90 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join4.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join4.q.out @@ -262,7 +262,7 @@ left outer join small_alltypesorc_b cd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### -{"optimizedSQL":"SELECT *\nFROM (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`) AS `t0` ON `t`.`cint` = `t0`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"HashTable Sink Operator":{"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"OperatorId:":"HASHTABLESINK_10"}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]"},"Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_12","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col1":"0:_col1","_col10":"0:_col10","_col11":"0:_col11","_col12":"1:_col0","_col13":"1:_col1","_col14":"1:_col2","_col15":"1:_col3","_col16":"1:_col4","_col17":"1:_col5","_col18":"1:_col6","_col19":"1:_col7","_col2":"0:_col2","_col20":"1:_col8","_col21":"1:_col9","_col22":"1:_col10","_col23":"1:_col11","_col3":"0:_col3","_col4":"0:_col4","_col5":"0:_col5","_col6":"0:_col6","_col7":"0:_col7","_col8":"0:_col8","_col9":"0:_col9"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint","col 1:smallint","col 2:int","col 3:bigint","col 4:float","col 5:double","col 6:string","col 7:string","col 8:timestamp","col 9:timestamp","col 10:boolean","col 11:boolean"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"],"Statistics:":"Num rows: 69 Data size: 38616 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_13","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 69 Data size: 38616 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_14"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint, bigint, bigint, double, double, string, string, timestamp, timestamp, bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_15"}}}}}} +{"optimizedSQL":"SELECT *\nFROM (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_10","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"OperatorId:":"HASHTABLESINK_12"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]"},"Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_14","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col1":"0:_col1","_col10":"0:_col10","_col11":"0:_col11","_col12":"1:_col0","_col13":"1:_col1","_col14":"1:_col2","_col15":"1:_col3","_col16":"1:_col4","_col17":"1:_col5","_col18":"1:_col6","_col19":"1:_col7","_col2":"0:_col2","_col20":"1:_col8","_col21":"1:_col9","_col22":"1:_col10","_col23":"1:_col11","_col3":"0:_col3","_col4":"0:_col4","_col5":"0:_col5","_col6":"0:_col6","_col7":"0:_col7","_col8":"0:_col8","_col9":"0:_col9"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint","col 1:smallint","col 2:int","col 3:bigint","col 4:float","col 5:double","col 6:string","col 7:string","col 8:timestamp","col 9:timestamp","col 10:boolean","col 11:boolean"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"],"Statistics:":"Num rows: 76 Data size: 34428 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_15","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 76 Data size: 34428 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint, bigint, bigint, double, double, string, string, timestamp, timestamp, bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} PREHOOK: query: select * from small_alltypesorc_b c left outer join small_alltypesorc_b cd @@ -347,7 +347,7 @@ left outer join small_alltypesorc_b hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### -{"optimizedSQL":"SELECT `t`.`ctinyint`\nFROM (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`) AS `t0` ON `t`.`ctinyint` = `t0`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_10"}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_12","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 900 Data size: 3564 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_13","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 900 Data size: 3564 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_14"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_15"}}}}}} +{"optimizedSQL":"SELECT `t`.`ctinyint`\nFROM (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_10","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_12"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_14","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_15","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} PREHOOK: query: select c.ctinyint from small_alltypesorc_b c left outer join small_alltypesorc_b hd @@ -794,7 +794,7 @@ left outer join small_alltypesorc_b hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### -{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `cint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_b`) AS `t0` ON `t`.`cint` = `t0`.`cint`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_26"}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_4","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_5","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","cint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), cint (type: int)","columnExprMap:":{"_col0":"ctinyint","_col1":"cint"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 2]"},"Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_28","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 69 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_29","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 2070 Data size: 16560 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_30","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_31","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_32"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 2]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_15","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_17"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_33"}}}}}} +{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `cint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t3` ON `t`.`ctinyint` = `t3`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","cint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), cint (type: int)","columnExprMap:":{"_col0":"ctinyint","_col1":"cint"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 2]"},"Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 58 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1160 Data size: 9280 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 2]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.ctinyint from small_alltypesorc_b c left outer join small_alltypesorc_b cd diff --git a/ql/src/test/results/clientpositive/vector_outer_join6.q.out b/ql/src/test/results/clientpositive/vector_outer_join6.q.out index bd938f631a..7269177c4b 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join6.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join6.q.out @@ -138,7 +138,7 @@ POSTHOOK: Input: default@tjoin1_n0 POSTHOOK: Input: default@tjoin2_n0 POSTHOOK: Input: default@tjoin3 #### A masked pattern was here #### -{"optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t0`.`rnum` AS `tj2rnum`, `t1`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`) AS `t0` ON `t`.`c1` = `t0`.`c1`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin3`) AS `t1` ON `t0`.`c1` = `t1`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_20"}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_4","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_5","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_18"}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_22","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_23","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2","_col4":"1:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col4"],"Statistics:":"Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_24","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int), _col4 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2","_col2":"_col4"},"outputColumnNames:":["_col0","_col1","_col2"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2]"},"Statistics:":"Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_25","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_26"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_27"}}}}}} +{"optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `tj2rnum`, `t3`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_16","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_22"}}}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2","_col4":"1:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col4"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_28","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int), _col4 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2","_col2":"_col4"},"outputColumnNames:":["_col0","_col1","_col2"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2]"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_29","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_30"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_31"}}}}}} PREHOOK: query: select tj1rnum, tj2rnum, tjoin3.rnum as rnumt3 from (select tjoin1_n0.rnum tj1rnum, tjoin2_n0.rnum tj2rnum, tjoin2_n0.c1 tj2c1 from tjoin1_n0 left outer join tjoin2_n0 on tjoin1_n0.c1 = tjoin2_n0.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1 PREHOOK: type: QUERY @@ -173,7 +173,7 @@ POSTHOOK: Input: default@tjoin1_n0 POSTHOOK: Input: default@tjoin2_n0 POSTHOOK: Input: default@tjoin3 #### A masked pattern was here #### -{"optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t0`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`) AS `t0` ON `t`.`c1` = `t0`.`c1`\nLEFT JOIN (SELECT `c1`\nFROM `default`.`tjoin3`) AS `t1` ON `t0`.`c1` = `t1`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_20"}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["c1"],"database:":"default","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_4","children":{"Select Operator":{"expressions:":"c1 (type: int)","columnExprMap:":{"_col0":"c1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_5","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_18"}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_22","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_23","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_24","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_25","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_26"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_27"}}}}}} +{"optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_16","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"c1 (type: int)","columnExprMap:":{"_col0":"c1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_22"}}}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_28","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_29","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_30"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_31"}}}}}} PREHOOK: query: select tj1rnum, tj2rnum as rnumt3 from (select tjoin1_n0.rnum tj1rnum, tjoin2_n0.rnum tj2rnum, tjoin2_n0.c1 tj2c1 from tjoin1_n0 left outer join tjoin2_n0 on tjoin1_n0.c1 = tjoin2_n0.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/vectorized_join46_mr.q.out b/ql/src/test/results/clientpositive/vectorized_join46_mr.q.out index 52107e91f8..5dbe87d28c 100644 --- a/ql/src/test/results/clientpositive/vectorized_join46_mr.q.out +++ b/ql/src/test/results/clientpositive/vectorized_join46_mr.q.out @@ -79,15 +79,19 @@ STAGE PLANS: $hdt$_1:test2 TableScan alias: test2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) Stage: Stage-3 Map Reduce @@ -117,13 +121,13 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -211,10 +215,10 @@ STAGE PLANS: $hdt$_1:test2 TableScan alias: test2 - filterExpr: key BETWEEN 100 AND 102 (type: boolean) + filterExpr: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: key BETWEEN 100 AND 102 (type: boolean) + predicate: (key BETWEEN 100 AND 102 and value is not null) (type: boolean) Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: int), value (type: int), col_2 (type: string) @@ -508,15 +512,19 @@ STAGE PLANS: $hdt$_0:test1 TableScan alias: test1 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) Stage: Stage-3 Map Reduce @@ -546,13 +554,13 @@ STAGE PLANS: nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Outer Join has keys IS true, Optimized Table and Supports Key Types IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1158,15 +1166,19 @@ STAGE PLANS: $hdt$_1:test2 TableScan alias: test2 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) Stage: Stage-3 Map Reduce @@ -1186,14 +1198,14 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col3 or _col4 BETWEEN 100 AND 102)} - Statistics: Num rows: 11 Data size: 1947 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1557 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 1903 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9 Data size: 1521 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1652,15 +1664,19 @@ STAGE PLANS: $hdt$_0:test1 TableScan alias: test1 + filterExpr: value is not null (type: boolean) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: int), value (type: int), col_1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col1 (type: int) Stage: Stage-3 Map Reduce @@ -1680,14 +1696,14 @@ STAGE PLANS: 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 residual filter predicates: {(_col0 BETWEEN 100 AND 102 or _col6)} - Statistics: Num rows: 10 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat