diff --git hbase-handler/src/test/results/positive/hbase_pushdown.q.out hbase-handler/src/test/results/positive/hbase_pushdown.q.out index 39c03eb..a5d8c6f 100644 --- hbase-handler/src/test/results/positive/hbase_pushdown.q.out +++ hbase-handler/src/test/results/positive/hbase_pushdown.q.out @@ -218,10 +218,10 @@ STAGE PLANS: alias: hbase_pushdown Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (((key = 80) and (key = 90)) and (value like '%90%')) (type: boolean) + predicate: false (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: 90 (type: int), value (type: string) + expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java index b1eca7d..1b327fe 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.optimizer.calcite; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -30,6 +31,7 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexShuttle; @@ -44,8 +46,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; public class HiveRexUtil { @@ -462,15 +466,33 @@ public static RexNode simplifyAnd2ForUnknownAsFalse(RexBuilder rexBuilder, return simplify(rexBuilder, terms.get(0), true); } // Try to simplify the expression + final Multimap> equalityTerms = ArrayListMultimap.create(); + final Map equalityConstantTerms = new HashMap<>(); final Set negatedTerms = new HashSet<>(); final Set nullOperands = new HashSet<>(); final Set notNullOperands = new LinkedHashSet<>(); final Set comparedOperands = new HashSet<>(); for (int i = 0; i < terms.size(); i++) { - final RexNode term = terms.get(i); + RexNode term = terms.get(i); if (!HiveCalciteUtil.isDeterministic(term)) { continue; } + // Simplify BOOLEAN expressions if possible + while (term.getKind() == SqlKind.EQUALS) { + RexCall call = (RexCall) term; + if (call.getOperands().get(0).isAlwaysTrue()) { + term = call.getOperands().get(1); + terms.remove(i); + terms.add(i, term); + continue; + } else if (call.getOperands().get(1).isAlwaysTrue()) { + term = call.getOperands().get(0); + terms.remove(i); + terms.add(i, term); + continue; + } + break; + } switch (term.getKind()) { case EQUALS: case NOT_EQUALS: @@ -481,18 +503,44 @@ public static RexNode simplifyAnd2ForUnknownAsFalse(RexBuilder rexBuilder, RexCall call = (RexCall) term; RexNode left = call.getOperands().get(0); comparedOperands.add(left.toString()); + RexCall leftCast = null; // if it is a cast, we include the inner reference if (left.getKind() == SqlKind.CAST) { - RexCall leftCast = (RexCall) left; + leftCast = (RexCall) left; comparedOperands.add(leftCast.getOperands().get(0).toString()); } RexNode right = call.getOperands().get(1); comparedOperands.add(right.toString()); + RexCall rightCast = null; // if it is a cast, we include the inner reference if (right.getKind() == SqlKind.CAST) { - RexCall rightCast = (RexCall) right; + rightCast = (RexCall) right; comparedOperands.add(rightCast.getOperands().get(0).toString()); } + // Check for equality on different constants. If the same ref or CAST(ref) + // is equal to different constants, this condition cannot be satisfied, + // and hence it can be evaluated to FALSE + if (term.getKind() == SqlKind.EQUALS) { + boolean leftRef = left instanceof RexInputRef || + (leftCast != null && leftCast.getOperands().get(0) instanceof RexInputRef); + boolean rightRef = right instanceof RexInputRef || + (rightCast != null && rightCast.getOperands().get(0) instanceof RexInputRef); + if (right instanceof RexLiteral && leftRef) { + final String literal = right.toString(); + final String prevLiteral = equalityConstantTerms.put(left.toString(), literal); + if (prevLiteral != null && !literal.equals(prevLiteral)) { + return rexBuilder.makeLiteral(false); + } + } else if (left instanceof RexLiteral && rightRef) { + final String literal = left.toString(); + final String prevLiteral = equalityConstantTerms.put(right.toString(), literal); + if (prevLiteral != null && !literal.equals(prevLiteral)) { + return rexBuilder.makeLiteral(false); + } + } else if (leftRef && rightRef) { + equalityTerms.put(left.toString(), Pair.of(right.toString(), term)); + } + } // Assume the expression a > 5 is part of a Filter condition. // Then we can derive the negated term: a <= 5. // But as the comparison is string based and thus operands order dependent, @@ -528,6 +576,30 @@ public static RexNode simplifyAnd2ForUnknownAsFalse(RexBuilder rexBuilder, if (!Collections.disjoint(nullOperands, comparedOperands)) { return rexBuilder.makeLiteral(false); } + // Check for equality of two refs wrt equality with constants + // Example #1. x=5 AND y=5 AND x=y : x=5 AND y=5 + // Example #2. x=5 AND y=6 AND x=y - not satisfiable + for (String ref1 : equalityTerms.keySet()) { + final String literal1 = equalityConstantTerms.get(ref1); + if (literal1 == null) { + continue; + } + Collection> references = equalityTerms.get(ref1); + for (Pair ref2 : references) { + final String literal2 = equalityConstantTerms.get(ref2.left); + if (literal2 == null) { + continue; + } + if (!literal1.equals(literal2)) { + // If an expression is equal to two different constants, + // it is not satisfiable + return rexBuilder.makeLiteral(false); + } + // Otherwise we can remove the term, as we already know that + // the expression is equal to two constants + terms.remove(ref2.right); + } + } // Remove not necessary IS NOT NULL expressions. // // Example. IS NOT NULL(x) AND x < 5 : x < 5 diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java index 514ae62..2fc68ae 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java @@ -31,7 +31,6 @@ import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.core.JoinInfo; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.rules.ValuesReduceRule; @@ -219,13 +218,7 @@ public JoinReduceExpressionsRule(Class joinClass, mq.getPulledUpPredicates(join.getRight()); final RelOptPredicateList predicates = leftPredicates.union(rightPredicates.shift(fieldCount)); - if (!reduceExpressions(join, expList, predicates)) { - return; - } - final JoinInfo joinInfo = JoinInfo.of(join.getLeft(), join.getRight(), expList.get(0)); - if (!joinInfo.isEqui()) { - // This kind of join must be an equi-join, and the condition is - // no longer an equi-join. SemiJoin is an example of this. + if (!reduceExpressions(join, expList, predicates, true)) { return; } call.transformTo( diff --git ql/src/test/results/clientpositive/annotate_stats_filter.q.out ql/src/test/results/clientpositive/annotate_stats_filter.q.out index 99183fc..a606e30 100644 --- ql/src/test/results/clientpositive/annotate_stats_filter.q.out +++ ql/src/test/results/clientpositive/annotate_stats_filter.q.out @@ -756,10 +756,10 @@ STAGE PLANS: alias: loc_orc Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((year = 2001) and (state = 'OH') and (state = 'FL')) (type: boolean) + predicate: false (type: boolean) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'FL' (type: string), locid (type: int), zip (type: bigint), 2001 (type: int) + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git ql/src/test/results/clientpositive/annotate_stats_part.q.out ql/src/test/results/clientpositive/annotate_stats_part.q.out index df42f36..50fc633 100644 --- ql/src/test/results/clientpositive/annotate_stats_part.q.out +++ ql/src/test/results/clientpositive/annotate_stats_part.q.out @@ -289,12 +289,12 @@ STAGE PLANS: alias: loc_orc Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator - predicate: ((year = '2001') and (year = '__HIVE_DEFAULT_PARTITION__')) (type: boolean) + predicate: false (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: state (type: string), locid (type: int), zip (type: bigint), '__HIVE_DEFAULT_PARTITION__' (type: string) + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- partition level partial column statistics diff --git ql/src/test/results/clientpositive/cbo_rp_join1.q.out ql/src/test/results/clientpositive/cbo_rp_join1.q.out index 4d785bf..9cb9594 100644 --- ql/src/test/results/clientpositive/cbo_rp_join1.q.out +++ ql/src/test/results/clientpositive/cbo_rp_join1.q.out @@ -119,7 +119,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### NULL -Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key = 40 AND a.value = 40 AND a.key = a.value AND b.key = 40 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key = 40 AND a.value = 40 AND a.key = a.value AND b.key = 40 @@ -141,7 +141,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 = _col1) and (_col1 = 40) and (_col0 = 40)) (type: boolean) + predicate: ((_col1 = 40) and (_col0 = 40)) (type: boolean) Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: @@ -224,6 +224,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### NULL +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key = 40 AND a.key = b.key AND b.key = 40 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key = 40 AND a.key = b.key AND b.key = 40 @@ -248,11 +249,9 @@ STAGE PLANS: predicate: (_col0 = 40) (type: boolean) Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + sort order: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) + value expressions: _col0 (type: int), _col1 (type: int) TableScan alias: a Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE @@ -264,24 +263,22 @@ STAGE PLANS: predicate: (_col0 = 40) (type: boolean) Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + sort order: Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) + value expressions: _col0 (type: int), _col1 (type: int) Reduce Operator Tree: Join Operator condition map: Outer Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + 0 + 1 outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 23 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hash(_col0,_col1,_col2,_col3) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 23 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col0) mode: hash @@ -322,6 +319,7 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key = 40 AND a.key = b.key AND b.key = 40 PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/constprog2.q.out ql/src/test/results/clientpositive/constprog2.q.out index 4200bf3..88679af 100644 --- ql/src/test/results/clientpositive/constprog2.q.out +++ ql/src/test/results/clientpositive/constprog2.q.out @@ -30,7 +30,7 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 86) (type: boolean) + predicate: (86 = key) (type: boolean) Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) diff --git ql/src/test/results/clientpositive/constprog_partitioner.q.out ql/src/test/results/clientpositive/constprog_partitioner.q.out index f66a1db..5b212f5 100644 --- ql/src/test/results/clientpositive/constprog_partitioner.q.out +++ ql/src/test/results/clientpositive/constprog_partitioner.q.out @@ -30,7 +30,7 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 100) (type: boolean) + predicate: (100 = key) (type: boolean) Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) diff --git ql/src/test/results/clientpositive/constprog_semijoin.q.out ql/src/test/results/clientpositive/constprog_semijoin.q.out index 1940987..040cfb4 100644 --- ql/src/test/results/clientpositive/constprog_semijoin.q.out +++ ql/src/test/results/clientpositive/constprog_semijoin.q.out @@ -421,26 +421,26 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid <> 100)) (type: boolean) - Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE + expressions: id (type: int), val (type: string), val1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: int), true (type: boolean) + key expressions: 100 (type: int), true (type: boolean) sort order: ++ - Map-reduce partition columns: _col3 (type: int), true (type: boolean) - Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: 100 (type: int), true (type: boolean) + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) TableScan alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id <> 100)) (type: boolean) + predicate: ((id = 100) and (id = 100) is not null) (type: boolean) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), true (type: boolean) + expressions: 100 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -458,13 +458,13 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 keys: - 0 _col3 (type: int), true (type: boolean) + 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -502,50 +502,50 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((dimid) IN (100, 200) and ((dimid = 100) = true)) (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + expressions: id (type: int), val (type: string), val1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: int), true (type: boolean) + key expressions: 100 (type: int), true (type: boolean) sort order: ++ - Map-reduce partition columns: _col3 (type: int), true (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: 100 (type: int), true (type: boolean) + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) TableScan alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((id) IN (100, 200) and ((id = 100) = true)) (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + predicate: ((id = 100) and (id = 100) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), true (type: boolean) + expressions: 100 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int), _col1 (type: boolean) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: boolean) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col3 (type: int), true (type: boolean) + 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -585,50 +585,50 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid = 200)) (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 200 (type: int), true (type: boolean) + key expressions: 100 (type: int), true (type: boolean) sort order: ++ - Map-reduce partition columns: 200 (type: int), true (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: 100 (type: int), true (type: boolean) + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) TableScan alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id = 200)) (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + predicate: ((id = 100) and (id = 100) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 200 (type: int), true (type: boolean) + expressions: 100 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int), _col1 (type: boolean) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: boolean) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 200 (type: int), true (type: boolean) + 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -666,38 +666,38 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid = 100)) (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 100 (type: int), true (type: boolean) sort order: ++ Map-reduce partition columns: 100 (type: int), true (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) TableScan alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id = 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + predicate: ((id = 100) and (id = 100) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 100 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int), _col1 (type: boolean) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: boolean) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -706,10 +706,10 @@ STAGE PLANS: 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -749,26 +749,26 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and dimid is not null) (type: boolean) + predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean) Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: id (type: int), val (type: string), val1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: int), true (type: boolean) + key expressions: 100 (type: int), true (type: boolean) sort order: ++ - Map-reduce partition columns: _col3 (type: int), true (type: boolean) + Map-reduce partition columns: 100 (type: int), true (type: boolean) Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) TableScan alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and id is not null) (type: boolean) + predicate: ((id = 100) and (id = 100) is not null) (type: boolean) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), true (type: boolean) + expressions: 100 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -786,7 +786,7 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 keys: - 0 _col3 (type: int), true (type: boolean) + 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/cte_5.q.out ql/src/test/results/clientpositive/cte_5.q.out index e9d700d..6bed6b1 100644 --- ql/src/test/results/clientpositive/cte_5.q.out +++ ql/src/test/results/clientpositive/cte_5.q.out @@ -63,6 +63,7 @@ src_thrift srcbucket srcbucket2 srcpart +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.colnum @@ -87,18 +88,13 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: colnum is not null (type: boolean) + predicate: (UDFToDouble(colnum) = 5.0) (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: colnum (type: int) - outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) + sort order: Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -108,26 +104,27 @@ STAGE PLANS: Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 5.0 (type: double) - sort order: + - Map-reduce partition columns: 5.0 (type: double) + sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 UDFToDouble(_col0) (type: double) - 1 5.0 (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 + 1 + Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 5 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -135,6 +132,7 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: with q1 as (select * from src where key= '5') select a.colnum from mydb.q1 as a join q1 as b diff --git ql/src/test/results/clientpositive/filter_cond_pushdown.q.out ql/src/test/results/clientpositive/filter_cond_pushdown.q.out index dc54bce..68eeeae 100644 --- ql/src/test/results/clientpositive/filter_cond_pushdown.q.out +++ ql/src/test/results/clientpositive/filter_cond_pushdown.q.out @@ -283,50 +283,50 @@ STAGE PLANS: alias: t2 Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE + predicate: (UDFToDouble(key) = 1.0) (type: boolean) + Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), c_int (type: int), c_float (type: float) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), UDFToDouble(_col0) (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), UDFToDouble(_col0) (type: double) - Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: float) TableScan alias: t3 Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((c_int = 1) and key is not null) (type: boolean) - Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE + predicate: ((c_int = 1) and (UDFToDouble(key) = 1.0)) (type: boolean) + Statistics: Num rows: 5 Data size: 65 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), c_float (type: float) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 65 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), 1.0 (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), 1.0 (type: double) - Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: float) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 5 Data size: 65 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: float) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string), UDFToDouble(_col0) (type: double) - 1 _col0 (type: string), 1.0 (type: double) - outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 22 Data size: 288 Basic stats: COMPLETE Column stats: NONE + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col4 + Statistics: Num rows: 11 Data size: 144 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col2 + _col5) > 2.0) or ((_col1 + 1) > 2)) (type: boolean) - Statistics: Num rows: 14 Data size: 183 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col2 + _col4) > 2.0) or ((_col1 + 1) > 2)) (type: boolean) + Statistics: Num rows: 6 Data size: 78 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 14 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 78 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -341,23 +341,23 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE + predicate: (UDFToDouble(key) = 1.0) (type: boolean) + Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 14 Data size: 183 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 78 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -366,10 +366,10 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 22 Data size: 288 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 144 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 22 Data size: 288 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 11 Data size: 144 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/join42.q.out ql/src/test/results/clientpositive/join42.q.out index 8fbcaed..d557385 100644 --- ql/src/test/results/clientpositive/join42.q.out +++ ql/src/test/results/clientpositive/join42.q.out @@ -144,7 +144,7 @@ STAGE PLANS: alias: la Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((loan_id = 4436) and aid is not null and pi_id is not null) (type: boolean) + predicate: ((4436 = loan_id) and aid is not null and pi_id is not null) (type: boolean) Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: aid (type: int), pi_id (type: int) @@ -158,7 +158,7 @@ STAGE PLANS: alias: fr Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (loan_id = 4436) (type: boolean) + predicate: (4436 = loan_id) (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/lineage3.q.out ql/src/test/results/clientpositive/lineage3.q.out index 4b6558b..af48d36 100644 --- ql/src/test/results/clientpositive/lineage3.q.out +++ ql/src/test/results/clientpositive/lineage3.q.out @@ -61,7 +61,7 @@ having min(cbigint) > 10 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Output: default@dest_l1@ds=tomorrow -{"version":"1.0","engine":"mr","database":"default","hash":"4ad6338a8abfe3fe0342198fcbd1f11d","queryText":"insert into table dest_l1 partition (ds='tomorrow')\nselect min(cint), cast(min(cstring1) as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cboolean1 = true\ngroup by csmallint\nhaving min(cbigint) > 10","edges":[{"sources":[2],"targets":[0],"expression":"min(default.alltypesorc.cint)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( min(default.alltypesorc.cstring1) AS varchar(128))","edgeType":"PROJECTION"},{"sources":[4,2],"targets":[0,1],"expression":"((alltypesorc.cboolean1 = true) and alltypesorc.cint is not null)","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1],"expression":"(min(default.alltypesorc.cbigint) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"4ad6338a8abfe3fe0342198fcbd1f11d","queryText":"insert into table dest_l1 partition (ds='tomorrow')\nselect min(cint), cast(min(cstring1) as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cboolean1 = true\ngroup by csmallint\nhaving min(cbigint) > 10","edges":[{"sources":[2],"targets":[0],"expression":"min(default.alltypesorc.cint)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( min(default.alltypesorc.cstring1) AS varchar(128))","edgeType":"PROJECTION"},{"sources":[4,2],"targets":[0,1],"expression":"(alltypesorc.cboolean1 and alltypesorc.cint is not null)","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1],"expression":"(min(default.alltypesorc.cbigint) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]} PREHOOK: query: select cint, rank() over(order by cint) from alltypesorc where cint > 10 and cint < 10000 limit 10 PREHOOK: type: QUERY @@ -317,7 +317,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@dest_v3 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"40bccc0722002f798d0548b59e369e83","queryText":"select * from dest_v3 limit 2","edges":[{"sources":[3,4,5,6,7],"targets":[0],"expression":"(tok_function sum (. (tok_table_or_col $hdt$_0) ctinyint) (tok_windowspec (tok_partitioningspec (tok_distributeby (. (tok_table_or_col $hdt$_0) csmallint)) (tok_orderby (tok_tabsortcolnameasc (tok_nulls_first (. (tok_table_or_col $hdt$_0) csmallint))))) (tok_windowvalues (preceding 2147483647) current)))","edgeType":"PROJECTION"},{"sources":[6],"targets":[1],"expression":"count(default.alltypesorc.cstring1)","edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"},{"sources":[8,7],"targets":[0,1,2],"expression":"((a.cboolean2 = true) and a.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(a.cint = a.cint)","edgeType":"PREDICATE"},{"sources":[9,7],"targets":[0,1,2],"expression":"((a.cfloat > 0) and a.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(count(default.alltypesorc.cint) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"dest_v3.a"},{"id":1,"vertexType":"COLUMN","vertexId":"dest_v3.x"},{"id":2,"vertexType":"COLUMN","vertexId":"dest_v3.cboolean1"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.csmallint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean2"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cfloat"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"40bccc0722002f798d0548b59e369e83","queryText":"select * from dest_v3 limit 2","edges":[{"sources":[3,4,5,6,7],"targets":[0],"expression":"(tok_function sum (. (tok_table_or_col $hdt$_0) ctinyint) (tok_windowspec (tok_partitioningspec (tok_distributeby (. (tok_table_or_col $hdt$_0) csmallint)) (tok_orderby (tok_tabsortcolnameasc (tok_nulls_first (. (tok_table_or_col $hdt$_0) csmallint))))) (tok_windowvalues (preceding 2147483647) current)))","edgeType":"PROJECTION"},{"sources":[6],"targets":[1],"expression":"count(default.alltypesorc.cstring1)","edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"},{"sources":[8,7],"targets":[0,1,2],"expression":"(a.cboolean2 and a.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(a.cint = a.cint)","edgeType":"PREDICATE"},{"sources":[9,7],"targets":[0,1,2],"expression":"((a.cfloat > 0) and a.cint is not null)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(count(default.alltypesorc.cint) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"dest_v3.a"},{"id":1,"vertexType":"COLUMN","vertexId":"dest_v3.x"},{"id":2,"vertexType":"COLUMN","vertexId":"dest_v3.cboolean1"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.csmallint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean2"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cfloat"}]} 38 216 false 38 229 true PREHOOK: query: drop table if exists src_dp diff --git ql/src/test/results/clientpositive/llap/cte_5.q.out ql/src/test/results/clientpositive/llap/cte_5.q.out index 9f9b718..b69f73d 100644 --- ql/src/test/results/clientpositive/llap/cte_5.q.out +++ ql/src/test/results/clientpositive/llap/cte_5.q.out @@ -63,6 +63,7 @@ src_thrift srcbucket srcbucket2 srcpart +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.colnum @@ -93,18 +94,13 @@ STAGE PLANS: alias: a Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: colnum is not null (type: boolean) + predicate: (UDFToDouble(colnum) = 5.0) (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: colnum (type: int) - outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) + sort order: Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs Map 3 @@ -118,9 +114,7 @@ STAGE PLANS: Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 5.0 (type: double) - sort order: + - Map-reduce partition columns: 5.0 (type: double) + sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -131,17 +125,20 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 UDFToDouble(_col0) (type: double) - 1 5.0 (type: double) - outputColumnNames: _col0 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 + 1 + Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 5 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -149,6 +146,7 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: with q1 as (select * from src where key= '5') select a.colnum from mydb.q1 as a join q1 as b diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out index b3673c2..88fca26 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out @@ -632,10 +632,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: d1 - filterExpr: (id = 1) (type: boolean) + filterExpr: (1 = id) (type: boolean) Statistics: Num rows: 3 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (id = 1) (type: boolean) + predicate: (1 = id) (type: boolean) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out index 3e96268..c91af30 100644 --- ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out @@ -1361,7 +1361,7 @@ STAGE PLANS: alias: l Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint = 6981) (type: boolean) + predicate: (6981 = cint) (type: boolean) Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cdecimal2 (type: decimal(23,14)) @@ -1559,7 +1559,7 @@ STAGE PLANS: alias: l Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint = 6981) (type: boolean) + predicate: (6981 = cint) (type: boolean) Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cdecimal2 (type: decimal(23,14)) diff --git ql/src/test/results/clientpositive/llap/tez_self_join.q.out ql/src/test/results/clientpositive/llap/tez_self_join.q.out index 50aa214..d3c9c70 100644 --- ql/src/test/results/clientpositive/llap/tez_self_join.q.out +++ ql/src/test/results/clientpositive/llap/tez_self_join.q.out @@ -102,7 +102,7 @@ STAGE PLANS: alias: self1 Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (id3 = 'ab') (type: boolean) + predicate: ('ab' = id3) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/mapjoin2.q.out ql/src/test/results/clientpositive/mapjoin2.q.out index 5ae1ac7..4b96ed3 100644 --- ql/src/test/results/clientpositive/mapjoin2.q.out +++ ql/src/test/results/clientpositive/mapjoin2.q.out @@ -26,6 +26,7 @@ POSTHOOK: Input: default@values__tmp__table__2 POSTHOOK: Output: default@tbl POSTHOOK: Lineage: tbl.n EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] POSTHOOK: Lineage: tbl.t SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: select a.n, a.t, isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a left outer join (select * from tbl where 1 = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl @@ -35,6 +36,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl #### A masked pattern was here #### 1 one true true +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: select isnull(a.n), isnull(a.t), b.n, b.t from (select * from tbl where 2 = 1) a right outer join (select * from tbl where n = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl @@ -44,6 +46,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl #### A masked pattern was here #### true true 2 two +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select isnull(a.n), isnull(a.t), isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a full outer join (select * from tbl where n = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl diff --git ql/src/test/results/clientpositive/mergejoin.q.out ql/src/test/results/clientpositive/mergejoin.q.out index 4b83a2d..6cd69db 100644 --- ql/src/test/results/clientpositive/mergejoin.q.out +++ ql/src/test/results/clientpositive/mergejoin.q.out @@ -2659,6 +2659,7 @@ POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### 480 +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@tab @@ -2678,6 +2679,7 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 0 val_0 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a right outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@tab @@ -2692,13 +2694,16 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 +Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join (select * from tab_part where tab_part.key = 98)b join tab_part c on a.key = b.key and b.key = c.key PREHOOK: type: QUERY PREHOOK: Input: default@tab +PREHOOK: Input: default@tab@ds=2008-04-08 PREHOOK: Input: default@tab_part +PREHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: select * from (select * from tab where tab.key = 0)a @@ -2706,9 +2711,12 @@ full outer join (select * from tab_part where tab_part.key = 98)b join tab_part c on a.key = b.key and b.key = c.key POSTHOOK: type: QUERY POSTHOOK: Input: default@tab +POSTHOOK: Input: default@tab@ds=2008-04-08 POSTHOOK: Input: default@tab_part +POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -2731,6 +2739,7 @@ NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 +Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a @@ -3258,12 +3267,15 @@ NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 +Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join (select * from tab_part where tab_part.key = 98)b on a.key = b.key full outer join tab_part c on b.key = c.key PREHOOK: type: QUERY PREHOOK: Input: default@tab +PREHOOK: Input: default@tab@ds=2008-04-08 PREHOOK: Input: default@tab_part PREHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### @@ -3273,6 +3285,7 @@ join (select * from tab_part where tab_part.key = 98)b on a.key = b.key full outer join tab_part c on b.key = c.key POSTHOOK: type: QUERY POSTHOOK: Input: default@tab +POSTHOOK: Input: default@tab@ds=2008-04-08 POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/ppd_outer_join5.q.out ql/src/test/results/clientpositive/ppd_outer_join5.q.out index cbc0e89..ab23d71 100644 --- ql/src/test/results/clientpositive/ppd_outer_join5.q.out +++ ql/src/test/results/clientpositive/ppd_outer_join5.q.out @@ -212,7 +212,7 @@ STAGE PLANS: alias: t3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (id = 20) (type: boolean) + predicate: (20 = id) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: 20 (type: int), key (type: string), value (type: string) @@ -320,7 +320,7 @@ STAGE PLANS: alias: t3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (id = 20) (type: boolean) + predicate: (20 = id) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: 20 (type: int), key (type: string), value (type: string) diff --git ql/src/test/results/clientpositive/ppd_udf_case.q.out ql/src/test/results/clientpositive/ppd_udf_case.q.out index 42f289e..4885dcd 100644 --- ql/src/test/results/clientpositive/ppd_udf_case.q.out +++ ql/src/test/results/clientpositive/ppd_udf_case.q.out @@ -52,7 +52,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((ds = '2008-04-08') and (key = '27')) (type: boolean) + predicate: ((ds = '2008-04-08') and ('27' = key)) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string), hr (type: string) @@ -196,7 +196,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = '27') (type: boolean) + predicate: ('27' = key) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string), hr (type: string) diff --git ql/src/test/results/clientpositive/ppd_union_view.q.out ql/src/test/results/clientpositive/ppd_union_view.q.out index 3081e28..aa123d3 100644 --- ql/src/test/results/clientpositive/ppd_union_view.q.out +++ ql/src/test/results/clientpositive/ppd_union_view.q.out @@ -502,7 +502,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((ds = '2011-10-15') and keymap is not null) (type: boolean) + predicate: (('2011-10-15' = ds) and keymap is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: key (type: string), keymap (type: string) diff --git ql/src/test/results/clientpositive/smb_mapjoin_25.q.out ql/src/test/results/clientpositive/smb_mapjoin_25.q.out index c91d6c3..75a0cbe 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_25.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_25.q.out @@ -80,7 +80,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) + predicate: (5 = key) (type: boolean) Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE @@ -151,7 +151,7 @@ STAGE PLANS: alias: d Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) + predicate: (5 = key) (type: boolean) Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE @@ -226,7 +226,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) + predicate: (5 = key) (type: boolean) Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE @@ -402,7 +402,7 @@ STAGE PLANS: alias: d Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) + predicate: (5 = key) (type: boolean) Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out index 69b2ba9..ecff9cc 100644 --- ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out +++ ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out @@ -37,7 +37,7 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 100) (type: boolean) + predicate: (100 = key) (type: boolean) Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) diff --git ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out index de829e2..c6a9b14 100644 --- ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out +++ ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out @@ -434,17 +434,17 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid <> 100)) (type: boolean) - Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE + expressions: id (type: int), val (type: string), val1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: int), true (type: boolean) + key expressions: 100 (type: int), true (type: boolean) sort order: ++ - Map-reduce partition columns: _col3 (type: int), true (type: boolean) - Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: 100 (type: int), true (type: boolean) + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) Map 3 Map Operator Tree: @@ -452,10 +452,10 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id <> 100)) (type: boolean) + predicate: ((id = 100) and (id = 100) is not null) (type: boolean) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), true (type: boolean) + expressions: 100 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -474,13 +474,13 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 keys: - 0 _col3 (type: int), true (type: boolean) + 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -523,17 +523,17 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((dimid) IN (100, 200) and ((dimid = 100) = true)) (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + expressions: id (type: int), val (type: string), val1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: int), true (type: boolean) + key expressions: 100 (type: int), true (type: boolean) sort order: ++ - Map-reduce partition columns: _col3 (type: int), true (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: 100 (type: int), true (type: boolean) + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) Map 3 Map Operator Tree: @@ -541,35 +541,35 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((id) IN (100, 200) and ((id = 100) = true)) (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + predicate: ((id = 100) and (id = 100) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), true (type: boolean) + expressions: 100 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int), _col1 (type: boolean) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: boolean) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col3 (type: int), true (type: boolean) + 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -614,17 +614,17 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid = 200)) (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 200 (type: int), true (type: boolean) + key expressions: 100 (type: int), true (type: boolean) sort order: ++ - Map-reduce partition columns: 200 (type: int), true (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: 100 (type: int), true (type: boolean) + Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) Map 3 Map Operator Tree: @@ -632,35 +632,35 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id = 200)) (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + predicate: ((id = 100) and (id = 100) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 200 (type: int), true (type: boolean) + expressions: 100 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int), _col1 (type: boolean) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: boolean) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 200 (type: int), true (type: boolean) + 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -703,17 +703,17 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid = 100)) (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean) + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 100 (type: int), true (type: boolean) sort order: ++ Map-reduce partition columns: 100 (type: int), true (type: boolean) - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) Map 3 Map Operator Tree: @@ -721,22 +721,22 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id = 100)) (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + predicate: ((id = 100) and (id = 100) is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 100 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int), _col1 (type: boolean) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: boolean) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -746,10 +746,10 @@ STAGE PLANS: 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -794,16 +794,16 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and dimid is not null) (type: boolean) + predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean) Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: id (type: int), val (type: string), val1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: int), true (type: boolean) + key expressions: 100 (type: int), true (type: boolean) sort order: ++ - Map-reduce partition columns: _col3 (type: int), true (type: boolean) + Map-reduce partition columns: 100 (type: int), true (type: boolean) Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) Map 3 @@ -812,10 +812,10 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and id is not null) (type: boolean) + predicate: ((id = 100) and (id = 100) is not null) (type: boolean) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), true (type: boolean) + expressions: 100 (type: int), true (type: boolean) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -834,7 +834,7 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 keys: - 0 _col3 (type: int), true (type: boolean) + 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out index ae266e5..8ad9ac5 100644 --- ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out +++ ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out @@ -197,7 +197,7 @@ STAGE PLANS: alias: t3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (id = 20) (type: boolean) + predicate: (20 = id) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: 20 (type: int), key (type: string), value (type: string) @@ -305,7 +305,7 @@ STAGE PLANS: alias: t3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: (id = 20) (type: boolean) + predicate: (20 = id) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: 20 (type: int), key (type: string), value (type: string) diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out index 96e7731..b38c182 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out @@ -87,7 +87,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) + predicate: (5 = key) (type: boolean) Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE @@ -113,7 +113,7 @@ STAGE PLANS: alias: d Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) + predicate: (5 = key) (type: boolean) Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE @@ -207,7 +207,7 @@ STAGE PLANS: alias: b Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) + predicate: (5 = key) (type: boolean) Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE @@ -274,7 +274,7 @@ STAGE PLANS: alias: d Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) + predicate: (5 = key) (type: boolean) Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out index 0f3bd63..15a8351 100644 --- ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out +++ ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out @@ -261,17 +261,17 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) - Conds:RS_8._col3, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] + Merge Join Operator [MERGEJOIN_17] (rows=2 width=3) + Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_8] - PartitionCols:_col3, true - Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_15] (rows=5 width=20) - predicate:(((dimid = 100) = true) and (dimid <> 100)) + PartitionCols:100, true + Select Operator [SEL_2] (rows=1 width=20) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_15] (rows=1 width=20) + predicate:false TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] + default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1"] <-Map 3 [SIMPLE_EDGE] SHUFFLE [RS_9] PartitionCols:_col0, _col1 @@ -280,7 +280,7 @@ Stage-0 Select Operator [SEL_5] (rows=2 width=3) Output:["_col0","_col1"] Filter Operator [FIL_16] (rows=2 width=3) - predicate:(((id = 100) = true) and (id <> 100)) + predicate:((id = 100) and (id = 100) is not null) TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] @@ -309,26 +309,26 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=2 width=22) - Conds:RS_8._col3, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] + Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) + Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_8] - PartitionCols:_col3, true - Select Operator [SEL_2] (rows=2 width=20) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_15] (rows=2 width=20) - predicate:((dimid) IN (100, 200) and ((dimid = 100) = true)) + PartitionCols:100, true + Select Operator [SEL_2] (rows=5 width=20) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_15] (rows=5 width=20) + predicate:((dimid = 100) and (dimid = 100) is not null) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] <-Map 3 [SIMPLE_EDGE] SHUFFLE [RS_9] PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=1 width=3) + Group By Operator [GBY_7] (rows=2 width=3) Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=1 width=3) + Select Operator [SEL_5] (rows=2 width=3) Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=1 width=3) - predicate:((id) IN (100, 200) and ((id = 100) = true)) + Filter Operator [FIL_16] (rows=2 width=3) + predicate:((id = 100) and (id = 100) is not null) TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] @@ -359,26 +359,26 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=2 width=22) - Conds:RS_8.200, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] + Merge Join Operator [MERGEJOIN_17] (rows=2 width=3) + Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_8] - PartitionCols:200, true - Select Operator [SEL_2] (rows=2 width=20) + PartitionCols:100, true + Select Operator [SEL_2] (rows=1 width=20) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=2 width=20) - predicate:(((dimid = 100) = true) and (dimid = 200)) + Filter Operator [FIL_15] (rows=1 width=20) + predicate:false TableScan [TS_0] (rows=10 width=20) - default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] + default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1"] <-Map 3 [SIMPLE_EDGE] SHUFFLE [RS_9] PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=1 width=3) + Group By Operator [GBY_7] (rows=2 width=3) Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=1 width=3) + Select Operator [SEL_5] (rows=2 width=3) Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=1 width=3) - predicate:(((id = 100) = true) and (id = 200)) + Filter Operator [FIL_16] (rows=2 width=3) + predicate:((id = 100) and (id = 100) is not null) TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] @@ -407,26 +407,26 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_17] (rows=2 width=22) + Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_8] PartitionCols:100, true - Select Operator [SEL_2] (rows=2 width=20) + Select Operator [SEL_2] (rows=5 width=20) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_15] (rows=2 width=20) - predicate:(((dimid = 100) = true) and (dimid = 100)) + Filter Operator [FIL_15] (rows=5 width=20) + predicate:((dimid = 100) and (dimid = 100) is not null) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] <-Map 3 [SIMPLE_EDGE] SHUFFLE [RS_9] PartitionCols:_col0, _col1 - Group By Operator [GBY_7] (rows=1 width=3) + Group By Operator [GBY_7] (rows=2 width=3) Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_5] (rows=1 width=3) + Select Operator [SEL_5] (rows=2 width=3) Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=1 width=3) - predicate:(((id = 100) = true) and (id = 100)) + Filter Operator [FIL_16] (rows=2 width=3) + predicate:((id = 100) and (id = 100) is not null) TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] @@ -458,14 +458,14 @@ Stage-0 Reducer 2 File Output Operator [FS_12] Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) - Conds:RS_8._col3, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] + Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_8] - PartitionCols:_col3, true + PartitionCols:100, true Select Operator [SEL_2] (rows=5 width=20) - Output:["_col0","_col1","_col2","_col3"] + Output:["_col0","_col1","_col2"] Filter Operator [FIL_15] (rows=5 width=20) - predicate:(((dimid = 100) = true) and dimid is not null) + predicate:((dimid = 100) and (dimid = 100) is not null) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] <-Map 3 [SIMPLE_EDGE] @@ -476,7 +476,7 @@ Stage-0 Select Operator [SEL_5] (rows=2 width=3) Output:["_col0","_col1"] Filter Operator [FIL_16] (rows=2 width=3) - predicate:(((id = 100) = true) and id is not null) + predicate:((id = 100) and (id = 100) is not null) TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] diff --git ql/src/test/results/clientpositive/tez/cte_5.q.out ql/src/test/results/clientpositive/tez/cte_5.q.out index 579b4f3..73342f5 100644 --- ql/src/test/results/clientpositive/tez/cte_5.q.out +++ ql/src/test/results/clientpositive/tez/cte_5.q.out @@ -63,6 +63,7 @@ src_thrift srcbucket srcbucket2 srcpart +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.colnum @@ -86,26 +87,26 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_10] - Merge Join Operator [MERGEJOIN_15] (rows=275 width=10) - Conds:RS_6.UDFToDouble(_col0)=RS_7.5.0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:UDFToDouble(_col0) - Select Operator [SEL_2] (rows=1 width=3) - Output:["_col0"] - Filter Operator [FIL_13] (rows=1 width=3) - predicate:colnum is not null - TableScan [TS_0] (rows=1 width=3) - mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:5.0 - Select Operator [SEL_5] (rows=250 width=10) - Filter Operator [FIL_14] (rows=250 width=10) - predicate:(key = '5') - TableScan [TS_3] (rows=500 width=10) - default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] + Select Operator [SEL_9] (rows=250 width=14) + Output:["_col0"] + Merge Join Operator [MERGEJOIN_13] (rows=250 width=14) + Conds:(Inner) + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_6] + Select Operator [SEL_2] (rows=1 width=3) + Filter Operator [FIL_11] (rows=1 width=3) + predicate:(UDFToDouble(colnum) = 5.0) + TableScan [TS_0] (rows=1 width=3) + mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"] + <-Map 3 [SIMPLE_EDGE] + SHUFFLE [RS_7] + Select Operator [SEL_5] (rows=250 width=10) + Filter Operator [FIL_12] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_3] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: with q1 as (select * from src where key= '5') select a.colnum from mydb.q1 as a join q1 as b diff --git ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out index 21cd9c8..f644e47 100644 --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out @@ -617,10 +617,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: d1 - filterExpr: (id = 1) (type: boolean) + filterExpr: (1 = id) (type: boolean) Statistics: Num rows: 3 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (id = 1) (type: boolean) + predicate: (1 = id) (type: boolean) Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out index 575d3da..53719d6 100644 --- ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out @@ -1313,7 +1313,7 @@ STAGE PLANS: alias: l Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint = 6981) (type: boolean) + predicate: (6981 = cint) (type: boolean) Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cdecimal2 (type: decimal(23,14)) @@ -1509,7 +1509,7 @@ STAGE PLANS: alias: l Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint = 6981) (type: boolean) + predicate: (6981 = cint) (type: boolean) Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cdecimal2 (type: decimal(23,14)) diff --git ql/src/test/results/clientpositive/tez/mapjoin2.q.out ql/src/test/results/clientpositive/tez/mapjoin2.q.out index 5ae1ac7..a7f7d15 100644 --- ql/src/test/results/clientpositive/tez/mapjoin2.q.out +++ ql/src/test/results/clientpositive/tez/mapjoin2.q.out @@ -26,6 +26,7 @@ POSTHOOK: Input: default@values__tmp__table__2 POSTHOOK: Output: default@tbl POSTHOOK: Lineage: tbl.n EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] POSTHOOK: Lineage: tbl.t SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: select a.n, a.t, isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a left outer join (select * from tbl where 1 = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl @@ -35,6 +36,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl #### A masked pattern was here #### 1 one true true +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 2' is a cross product PREHOOK: query: select isnull(a.n), isnull(a.t), b.n, b.t from (select * from tbl where 2 = 1) a right outer join (select * from tbl where n = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl @@ -44,6 +46,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl #### A masked pattern was here #### true true 2 two +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select isnull(a.n), isnull(a.t), isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a full outer join (select * from tbl where n = 2) b on a.n = b.n PREHOOK: type: QUERY PREHOOK: Input: default@tbl diff --git ql/src/test/results/clientpositive/tez/mergejoin.q.out ql/src/test/results/clientpositive/tez/mergejoin.q.out index 03d282c..56de8bb 100644 --- ql/src/test/results/clientpositive/tez/mergejoin.q.out +++ ql/src/test/results/clientpositive/tez/mergejoin.q.out @@ -2644,6 +2644,7 @@ POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### 480 +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@tab @@ -2663,6 +2664,7 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 0 val_0 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a right outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@tab @@ -2677,13 +2679,16 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join (select * from tab_part where tab_part.key = 98)b join tab_part c on a.key = b.key and b.key = c.key PREHOOK: type: QUERY PREHOOK: Input: default@tab +PREHOOK: Input: default@tab@ds=2008-04-08 PREHOOK: Input: default@tab_part +PREHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: select * from (select * from tab where tab.key = 0)a @@ -2691,9 +2696,12 @@ full outer join (select * from tab_part where tab_part.key = 98)b join tab_part c on a.key = b.key and b.key = c.key POSTHOOK: type: QUERY POSTHOOK: Input: default@tab +POSTHOOK: Input: default@tab@ds=2008-04-08 POSTHOOK: Input: default@tab_part +POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -2717,6 +2725,7 @@ NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 Warning: Shuffle Join MERGEJOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join @@ -3243,12 +3252,15 @@ NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08 +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join (select * from tab_part where tab_part.key = 98)b on a.key = b.key full outer join tab_part c on b.key = c.key PREHOOK: type: QUERY PREHOOK: Input: default@tab +PREHOOK: Input: default@tab@ds=2008-04-08 PREHOOK: Input: default@tab_part PREHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### @@ -3258,6 +3270,7 @@ join (select * from tab_part where tab_part.key = 98)b on a.key = b.key full outer join tab_part c on b.key = c.key POSTHOOK: type: QUERY POSTHOOK: Input: default@tab +POSTHOOK: Input: default@tab@ds=2008-04-08 POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/tez/tez_self_join.q.out ql/src/test/results/clientpositive/tez/tez_self_join.q.out index 1ec5048..0b51c1d 100644 --- ql/src/test/results/clientpositive/tez/tez_self_join.q.out +++ ql/src/test/results/clientpositive/tez/tez_self_join.q.out @@ -100,7 +100,7 @@ STAGE PLANS: alias: self1 Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (id3 = 'ab') (type: boolean) + predicate: ('ab' = id3) (type: boolean) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE