diff --git hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out index 27446b4..0ef0efd 100644 --- hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out +++ hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out @@ -438,6 +438,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hbase_pushdown + filterExpr: (key >= '90') (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: (key >= '90') (type: boolean) diff --git hbase-handler/src/test/results/positive/hbase_queries.q.out hbase-handler/src/test/results/positive/hbase_queries.q.out index a99f561..8aa5f84 100644 --- hbase-handler/src/test/results/positive/hbase_queries.q.out +++ hbase-handler/src/test/results/positive/hbase_queries.q.out @@ -917,7 +917,12 @@ WITH SERDEPROPERTIES ( 'hbase.columns.mapping'='cf:string', 'serialization.format'='1') TBLPROPERTIES ( + 'COLUMN_STATS_ACCURATE'='{\"BASIC_STATS\":\"true\"}', 'hbase.table.name'='hbase_table_0', + 'numFiles'='0', + 'numRows'='0', + 'rawDataSize'='0', + 'totalSize'='0', #### A masked pattern was here #### PREHOOK: query: DROP TABLE IF EXISTS hbase_table_9 PREHOOK: type: DROPTABLE diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java index bc52f7b..89de234 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java @@ -216,6 +216,10 @@ public ConstantPropagateProcCtx(ConstantPropagateOption option) { Operator parent = op.getParentOperators().get(0); if (op.getColumnExprMap() != null && op.getColumnExprMap().entrySet() != null) { for (Entry entry : op.getColumnExprMap().entrySet()) { + if (op.getSchema().getPosition(entry.getKey()) == -1) { + // Not present + continue; + } ExprNodeDesc expr = entry.getValue(); if (expr instanceof ExprNodeColumnDesc) { String parentColName = ((ExprNodeColumnDesc) expr).getColumn(); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java index 37dbe32..de4d0e4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java @@ -28,7 +28,6 @@ import java.util.Set; import java.util.Stack; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; @@ -109,7 +108,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (cSEL.getColumnExprMap() == null) { // If the child SelectOperator does not have the ColumnExprMap, // we do not need to update the ColumnExprMap in the parent SelectOperator. - pSEL.getConf().setColList(ExprNodeDescUtils.backtrack(cSELColList, cSEL, pSEL)); + pSEL.getConf().setColList(ExprNodeDescUtils.backtrack(cSELColList, cSEL, pSEL, true)); pSEL.getConf().setOutputColumnNames(cSELOutputColumnNames); } else { // If the child SelectOperator has the ColumnExprMap, @@ -121,7 +120,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, String outputColumnName = cSELOutputColumnNames.get(i); ExprNodeDesc cSELExprNodeDesc = cSELColList.get(i); ExprNodeDesc newPSELExprNodeDesc = - ExprNodeDescUtils.backtrack(cSELExprNodeDesc, cSEL, pSEL); + ExprNodeDescUtils.backtrack(cSELExprNodeDesc, cSEL, pSEL, true); newPSELColList.add(newPSELExprNodeDesc); newPSELOutputColumnNames.add(outputColumnName); colExprMap.put(outputColumnName, newPSELExprNodeDesc); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index bf9a0a3..5ee54b9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -129,13 +129,11 @@ public void initialize(HiveConf hiveConf) { /* Add list bucketing pruner. */ transformations.add(new ListBucketingPruner()); } - } - if ((HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD) - && HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION)) || - (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION) - && pctx.getContext().isCboSucceeded())) { - // PartitionPruner may create more folding opportunities, run ConstantPropagate again. - transformations.add(new ConstantPropagate()); + if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION) && + !pctx.getContext().isCboSucceeded()) { + // PartitionPruner may create more folding opportunities, run ConstantPropagate again. + transformations.add(new ConstantPropagate()); + } } if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTGROUPBY) || diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index 4adf7b2..36b7036 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.ListBucketingCtx; @@ -287,7 +288,7 @@ private boolean allStaticPartitions(Operator op, } if (op.getColumnExprMap() != null) { for(String dpCol : dpCols) { - ExprNodeDesc end = findConstantExprOrigin(dpCol, op); + ExprNodeDesc end = ExprNodeDescUtils.findConstantExprOrigin(dpCol, op); if (!(end instanceof ExprNodeConstantDesc)) { return false; } @@ -298,37 +299,6 @@ private boolean allStaticPartitions(Operator op, return true; } - // Find the constant origin of a certain column if it is originated from a constant - // Otherwise, it returns the expression that originated the column - private ExprNodeDesc findConstantExprOrigin(String dpCol, Operator op) { - ExprNodeDesc expr = op.getColumnExprMap().get(dpCol); - ExprNodeDesc foldedExpr; - // If it is a function, we try to fold it - if (expr instanceof ExprNodeGenericFuncDesc) { - foldedExpr = ConstantPropagateProcFactory.foldExpr((ExprNodeGenericFuncDesc)expr); - if (foldedExpr == null) { - foldedExpr = expr; - } - } else { - foldedExpr = expr; - } - // If it is a column reference, we will try to resolve it - if (foldedExpr instanceof ExprNodeColumnDesc) { - Operator originOp = null; - for(Operator parentOp : op.getParentOperators()) { - if (parentOp.getColumnExprMap() != null) { - originOp = parentOp; - break; - } - } - if (originOp != null) { - return findConstantExprOrigin(((ExprNodeColumnDesc)foldedExpr).getColumn(), originOp); - } - } - // Otherwise, we return the expression - return foldedExpr; - } - // Remove RS and SEL introduced by enforce bucketing/sorting config // Convert PARENT -> RS -> SEL -> FS to PARENT -> FS private boolean removeRSInsertedByEnforceBucketing(FileSinkOperator fsOp) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index 0cfd529..7febfd5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -22,7 +22,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -30,8 +29,6 @@ import java.util.Set; import java.util.Stack; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; @@ -67,6 +64,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCount; @@ -81,6 +79,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; @@ -213,13 +213,17 @@ private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { private boolean hasNullOrConstantGbyKey(GroupByOperator gbyOp) { GroupByDesc gbyDesc = gbyOp.getConf(); + int numCols = gbyDesc.getOutputColumnNames().size(); + int aggCols = gbyDesc.getAggregators().size(); // If the Group by operator has null key - if (gbyDesc.getOutputColumnNames().size() == - gbyDesc.getAggregators().size()) { + if (numCols == aggCols) { return true; } - for (ExprNodeDesc en :gbyDesc.getKeys()) { - if (!(en instanceof ExprNodeConstantDesc)) { + // If the Gby key is a constant + List dpCols = gbyOp.getSchema().getColumnNames().subList(0, numCols - aggCols); + for(String dpCol : dpCols) { + ExprNodeDesc end = ExprNodeDescUtils.findConstantExprOrigin(dpCol, gbyOp); + if (!(end instanceof ExprNodeConstantDesc)) { return false; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java index c6d1d46..4c154d0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java @@ -35,12 +35,12 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.util.ImmutableBitSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class HiveRelOptUtil extends RelOptUtil { @@ -105,23 +105,6 @@ private static void splitJoinCondition( final RelOptCluster cluster = inputs.get(0).getCluster(); final RexBuilder rexBuilder = cluster.getRexBuilder(); - final ImmutableBitSet[] inputsRange = new ImmutableBitSet[inputs.size()]; - int totalFieldCount = 0; - for (int i = 0; i < inputs.size(); i++) { - final int firstField = totalFieldCount + sysFieldCount; - totalFieldCount = firstField + inputs.get(i).getRowType().getFieldCount(); - inputsRange[i] = ImmutableBitSet.range(firstField, totalFieldCount); - } - - // adjustment array - int[] adjustments = new int[totalFieldCount]; - for (int i = 0; i < inputs.size(); i++) { - final int adjustment = inputsRange[i].nextSetBit(0); - for (int j = adjustment; j < inputsRange[i].length(); j++) { - adjustments[j] = -adjustment; - } - } - if (condition instanceof RexCall) { RexCall call = (RexCall) condition; if (call.getOperator() == SqlStdOperatorTable.AND) { @@ -165,6 +148,14 @@ private static void splitJoinCondition( final ImmutableBitSet projRefs0 = InputFinder.bits(op0); final ImmutableBitSet projRefs1 = InputFinder.bits(op1); + final ImmutableBitSet[] inputsRange = new ImmutableBitSet[inputs.size()]; + int totalFieldCount = 0; + for (int i = 0; i < inputs.size(); i++) { + final int firstField = totalFieldCount + sysFieldCount; + totalFieldCount = firstField + inputs.get(i).getRowType().getFieldCount(); + inputsRange[i] = ImmutableBitSet.range(firstField, totalFieldCount); + } + boolean foundBothInputs = false; for (int i = 0; i < inputs.size() && !foundBothInputs; i++) { if (projRefs0.intersects(inputsRange[i]) @@ -196,6 +187,15 @@ private static void splitJoinCondition( } if ((leftKey != null) && (rightKey != null)) { + // adjustment array + int[] adjustments = new int[totalFieldCount]; + for (int i = 0; i < inputs.size(); i++) { + final int adjustment = inputsRange[i].nextSetBit(0); + for (int j = adjustment; j < inputsRange[i].length(); j++) { + adjustments[j] = -adjustment; + } + } + // replace right Key input ref rightKey = rightKey.accept( diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java index 2f309f3..7186088 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java @@ -19,8 +19,11 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.calcite.linq4j.Ord; @@ -29,12 +32,16 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -42,6 +49,8 @@ public class HiveRexUtil { + protected static final Logger LOG = LoggerFactory.getLogger(HiveRexUtil.class); + /** * Simplifies a boolean expression. * @@ -54,13 +63,20 @@ * */ public static RexNode simplify(RexBuilder rexBuilder, RexNode e) { + return simplify(rexBuilder, e, false); + } + + public static RexNode simplify(RexBuilder rexBuilder, RexNode e, + boolean unknownAsFalse) { switch (e.getKind()) { case AND: - return simplifyAnd(rexBuilder, (RexCall) e); + return simplifyAnd(rexBuilder, (RexCall) e, unknownAsFalse); case OR: return simplifyOr(rexBuilder, (RexCall) e); + case NOT: + return simplifyNot(rexBuilder, (RexCall) e); case CASE: - return simplifyCase(rexBuilder, (RexCall) e); + return simplifyCase(rexBuilder, (RexCall) e, unknownAsFalse); case IS_NULL: return ((RexCall) e).getOperands().get(0).getType().isNullable() ? e : rexBuilder.makeLiteral(false); @@ -72,9 +88,32 @@ public static RexNode simplify(RexBuilder rexBuilder, RexNode e) { } } - private static RexNode simplifyCase(RexBuilder rexBuilder, RexCall call) { + private static RexNode simplifyNot(RexBuilder rexBuilder, RexCall call) { + final RexNode a = call.getOperands().get(0); + switch (a.getKind()) { + case NOT: + // NOT NOT x ==> x + return simplify(rexBuilder, ((RexCall) a).getOperands().get(0)); + } + final SqlKind negateKind = a.getKind().negate(); + if (a.getKind() != negateKind) { + return simplify(rexBuilder, + rexBuilder.makeCall(op(negateKind), + ImmutableList.of(((RexCall) a).getOperands().get(0)))); + } + final SqlKind negateKind2 = negate(a.getKind()); + if (a.getKind() != negateKind2) { + return simplify(rexBuilder, + rexBuilder.makeCall(op(negateKind2), ((RexCall) a).getOperands())); + } + return call; + } + + private static RexNode simplifyCase(RexBuilder rexBuilder, RexCall call, + boolean unknownAsFalse) { final List operands = call.getOperands(); final List newOperands = new ArrayList<>(); + final Set values = new HashSet<>(); for (int i = 0; i < operands.size(); i++) { RexNode operand = operands.get(i); if (RexUtil.isCasePredicate(call, i)) { @@ -88,13 +127,18 @@ private static RexNode simplifyCase(RexBuilder rexBuilder, RexCall call) { ++i; continue; } + } else { + if (unknownAsFalse && RexUtil.isNull(operand)) { + values.add(rexBuilder.makeLiteral(false).toString()); + } else { + values.add(operand.toString()); + } } newOperands.add(operand); } assert newOperands.size() % 2 == 1; - switch (newOperands.size()) { - case 1: - return rexBuilder.makeCast(call.getType(), newOperands.get(0)); + if (newOperands.size() == 1 || values.size() == 1) { + return rexBuilder.makeCast(call.getType(), newOperands.get(newOperands.size() - 1)); } trueFalse: if (call.getType().getSqlTypeName() == SqlTypeName.BOOLEAN) { @@ -110,7 +154,8 @@ private static RexNode simplifyCase(RexBuilder rexBuilder, RexCall call) { casePairs(rexBuilder, newOperands); for (Ord> pair : Ord.zip(pairs)) { if (!pair.e.getValue().isAlwaysTrue() - && !pair.e.getValue().isAlwaysFalse()) { + && !pair.e.getValue().isAlwaysFalse() + && (!unknownAsFalse || !RexUtil.isNull(pair.e.getValue()))) { break trueFalse; } } @@ -145,33 +190,76 @@ private static RexNode simplifyCase(RexBuilder rexBuilder, RexCall call) { return builder.build(); } - public static RexNode simplifyAnd(RexBuilder rexBuilder, RexCall e) { - final List terms = RelOptUtil.conjunctions(e); + public static RexNode simplifyAnd(RexBuilder rexBuilder, RexCall e, + boolean unknownAsFalse) { + final List terms = new ArrayList<>(); final List notTerms = new ArrayList<>(); - final List negatedTerms = new ArrayList<>(); - final List nullOperands = new ArrayList<>(); - final List notNullOperands = new ArrayList<>(); - final Set comparedOperands = new HashSet<>(); + RelOptUtil.decomposeConjunction(e, terms, notTerms); + if (unknownAsFalse) { + return simplifyAnd2ForUnknownAsFalse(rexBuilder, terms, notTerms); + } + return simplifyAnd2(rexBuilder, terms, notTerms); + } + + public static RexNode simplifyAnd2(RexBuilder rexBuilder, + List terms, List notTerms) { + if (terms.contains(rexBuilder.makeLiteral(false))) { + return rexBuilder.makeLiteral(false); + } + if (terms.isEmpty() && notTerms.isEmpty()) { + return rexBuilder.makeLiteral(true); + } + if (terms.size() == 1 && notTerms.isEmpty()) { + // Make sure "x OR y OR x" (a single-term conjunction) gets simplified. + return simplify(rexBuilder, terms.get(0)); + } + // If one of the not-disjunctions is a disjunction that is wholly + // contained in the disjunctions list, the expression is not + // satisfiable. + // + // Example #1. x AND y AND z AND NOT (x AND y) - not satisfiable + // Example #2. x AND y AND NOT (x AND y) - not satisfiable + // Example #3. x AND y AND NOT (x AND y AND z) - may be satisfiable + for (RexNode notDisjunction : notTerms) { + final List terms2 = RelOptUtil.conjunctions(notDisjunction); + if (terms.containsAll(terms2)) { + return rexBuilder.makeLiteral(false); + } + } + // Add the NOT disjunctions back in. + for (RexNode notDisjunction : notTerms) { + terms.add( + simplify(rexBuilder, + rexBuilder.makeCall(SqlStdOperatorTable.NOT, notDisjunction))); + } + return RexUtil.composeConjunction(rexBuilder, terms, false); + } + + /** As {@link #simplifyAnd2(RexBuilder, List, List)} but we assume that if the expression returns + * UNKNOWN it will be interpreted as FALSE. */ + public static RexNode simplifyAnd2ForUnknownAsFalse(RexBuilder rexBuilder, + List terms, List notTerms) { + if (terms.contains(rexBuilder.makeLiteral(false))) { + return rexBuilder.makeLiteral(false); + } + if (terms.isEmpty() && notTerms.isEmpty()) { + return rexBuilder.makeLiteral(true); + } + if (terms.size() == 1 && notTerms.isEmpty()) { + // Make sure "x OR y OR x" (a single-term conjunction) gets simplified. + return simplify(rexBuilder, terms.get(0), true); + } + // Try to simplify the expression + final Set negatedTerms = new HashSet<>(); + final Set nullOperands = new HashSet<>(); + final Set notNullOperands = new LinkedHashSet<>(); + final Set comparedOperands = new HashSet<>(); for (int i = 0; i < terms.size(); i++) { final RexNode term = terms.get(i); if (!HiveCalciteUtil.isDeterministic(term)) { continue; } switch (term.getKind()) { - case NOT: - notTerms.add( - ((RexCall) term).getOperands().get(0)); - terms.remove(i); - --i; - break; - case LITERAL: - if (!RexLiteral.booleanValue(term)) { - return term; // false - } else { - terms.remove(i); - --i; - } - break; case EQUALS: case NOT_EQUALS: case LESS_THAN: @@ -180,53 +268,48 @@ public static RexNode simplifyAnd(RexBuilder rexBuilder, RexCall e) { case GREATER_THAN_OR_EQUAL: RexCall call = (RexCall) term; RexNode left = call.getOperands().get(0); - comparedOperands.add(left); + comparedOperands.add(left.toString()); // if it is a cast, we include the inner reference if (left.getKind() == SqlKind.CAST) { RexCall leftCast = (RexCall) left; - comparedOperands.add(leftCast.getOperands().get(0)); + comparedOperands.add(leftCast.getOperands().get(0).toString()); } RexNode right = call.getOperands().get(1); - comparedOperands.add(right); + comparedOperands.add(right.toString()); // if it is a cast, we include the inner reference if (right.getKind() == SqlKind.CAST) { RexCall rightCast = (RexCall) right; - comparedOperands.add(rightCast.getOperands().get(0)); + comparedOperands.add(rightCast.getOperands().get(0).toString()); } - // Assume we have the expression a > 5. - // Then we can derive the negated term: NOT(a <= 5). + // Assume the expression a > 5 is part of a Filter condition. + // Then we can derive the negated term: a <= 5. // But as the comparison is string based and thus operands order dependent, - // we should also add the inverted negated term: NOT(5 >= a). + // we should also add the inverted negated term: 5 >= a. // Observe that for creating the inverted term we invert the list of operands. - RexCall negatedTerm = negate(rexBuilder, call); + RexNode negatedTerm = negate(rexBuilder, call); if (negatedTerm != null) { - negatedTerms.add(negatedTerm); - RexCall invertNegatedTerm = invert(rexBuilder, negatedTerm); + negatedTerms.add(negatedTerm.toString()); + RexNode invertNegatedTerm = invert(rexBuilder, (RexCall) negatedTerm); if (invertNegatedTerm != null) { - negatedTerms.add(invertNegatedTerm); + negatedTerms.add(invertNegatedTerm.toString()); } } break; case IN: - comparedOperands.add(((RexCall) term).operands.get(0)); + comparedOperands.add(((RexCall) term).operands.get(0).toString()); break; case BETWEEN: - comparedOperands.add(((RexCall) term).operands.get(1)); + comparedOperands.add(((RexCall) term).operands.get(1).toString()); break; case IS_NOT_NULL: - notNullOperands.add( - ((RexCall) term).getOperands().get(0)); + notNullOperands.add(((RexCall) term).getOperands().get(0)); terms.remove(i); --i; break; case IS_NULL: - nullOperands.add( - ((RexCall) term).getOperands().get(0)); + nullOperands.add(((RexCall) term).getOperands().get(0).toString()); } } - if (terms.isEmpty() && notTerms.isEmpty() && notNullOperands.isEmpty()) { - return rexBuilder.makeLiteral(true); - } // If one column should be null and is in a comparison predicate, // it is not satisfiable. // Example. IS NULL(x) AND x < 5 - not satisfiable @@ -237,10 +320,9 @@ public static RexNode simplifyAnd(RexBuilder rexBuilder, RexCall e) { // // Example. IS NOT NULL(x) AND x < 5 : x < 5 for (RexNode operand : notNullOperands) { - if (!comparedOperands.contains(operand)) { + if (!comparedOperands.contains(operand.toString())) { terms.add( - rexBuilder.makeCall( - SqlStdOperatorTable.IS_NOT_NULL, operand)); + rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, operand)); } } // If one of the not-disjunctions is a disjunction that is wholly @@ -253,23 +335,24 @@ public static RexNode simplifyAnd(RexBuilder rexBuilder, RexCall e) { final Set termsSet = new HashSet( Lists.transform(terms, HiveCalciteUtil.REX_STR_FN)); for (RexNode notDisjunction : notTerms) { - final Set notSet = new HashSet( - Lists.transform(RelOptUtil.conjunctions(notDisjunction), HiveCalciteUtil.REX_STR_FN)); - if (termsSet.containsAll(notSet)) { + if (!HiveCalciteUtil.isDeterministic(notDisjunction)) { + continue; + } + final List terms2Set = Lists.transform( + RelOptUtil.conjunctions(notDisjunction), HiveCalciteUtil.REX_STR_FN); + if (termsSet.containsAll(terms2Set)) { return rexBuilder.makeLiteral(false); } } // Add the NOT disjunctions back in. for (RexNode notDisjunction : notTerms) { terms.add( - rexBuilder.makeCall( - SqlStdOperatorTable.NOT, notDisjunction)); - } - // The negated terms - for (RexNode notDisjunction : negatedTerms) { - final Set notSet = new HashSet( - Lists.transform(RelOptUtil.conjunctions(notDisjunction), HiveCalciteUtil.REX_STR_FN)); - if (termsSet.containsAll(notSet)) { + simplify(rexBuilder, + rexBuilder.makeCall(SqlStdOperatorTable.NOT, notDisjunction), true)); + } + // The negated terms: only deterministic expressions + for (String negatedTerm : negatedTerms) { + if (termsSet.contains(negatedTerm)) { return rexBuilder.makeLiteral(false); } } @@ -284,11 +367,13 @@ public static RexNode simplifyOr(RexBuilder rexBuilder, RexCall call) { final RexNode term = terms.get(i); switch (term.getKind()) { case LITERAL: - if (RexLiteral.booleanValue(term)) { - return term; // true - } else { - terms.remove(i); - --i; + if (!RexLiteral.isNullLiteral(term)) { + if (RexLiteral.booleanValue(term)) { + return term; // true + } else { + terms.remove(i); + --i; + } } } } @@ -298,21 +383,34 @@ public static RexNode simplifyOr(RexBuilder rexBuilder, RexCall call) { private static RexCall negate(RexBuilder rexBuilder, RexCall call) { switch (call.getKind()) { case EQUALS: - return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS, call.getOperands()); case NOT_EQUALS: - return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, call.getOperands()); case LESS_THAN: - return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, call.getOperands()); case GREATER_THAN: - return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, call.getOperands()); case LESS_THAN_OR_EQUAL: - return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN, call.getOperands()); case GREATER_THAN_OR_EQUAL: - return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, call.getOperands()); + return (RexCall) rexBuilder.makeCall(op(negate(call.getKind())), call.getOperands()); } return null; } + private static SqlKind negate(SqlKind kind) { + switch (kind) { + case EQUALS: + return SqlKind.NOT_EQUALS; + case NOT_EQUALS: + return SqlKind.EQUALS; + case LESS_THAN: + return SqlKind.GREATER_THAN_OR_EQUAL; + case GREATER_THAN: + return SqlKind.LESS_THAN_OR_EQUAL; + case LESS_THAN_OR_EQUAL: + return SqlKind.GREATER_THAN; + case GREATER_THAN_OR_EQUAL: + return SqlKind.LESS_THAN; + } + return kind; + } + private static RexCall invert(RexBuilder rexBuilder, RexCall call) { switch (call.getKind()) { case LESS_THAN: @@ -330,4 +428,85 @@ private static RexCall invert(RexBuilder rexBuilder, RexCall call) { } return null; } + + private static SqlOperator op(SqlKind kind) { + switch (kind) { + case IS_FALSE: + return SqlStdOperatorTable.IS_FALSE; + case IS_TRUE: + return SqlStdOperatorTable.IS_TRUE; + case IS_UNKNOWN: + return SqlStdOperatorTable.IS_UNKNOWN; + case IS_NULL: + return SqlStdOperatorTable.IS_NULL; + case IS_NOT_FALSE: + return SqlStdOperatorTable.IS_NOT_FALSE; + case IS_NOT_TRUE: + return SqlStdOperatorTable.IS_NOT_TRUE; + case IS_NOT_NULL: + return SqlStdOperatorTable.IS_NOT_NULL; + case EQUALS: + return SqlStdOperatorTable.EQUALS; + case NOT_EQUALS: + return SqlStdOperatorTable.NOT_EQUALS; + case LESS_THAN: + return SqlStdOperatorTable.LESS_THAN; + case GREATER_THAN: + return SqlStdOperatorTable.GREATER_THAN; + case LESS_THAN_OR_EQUAL: + return SqlStdOperatorTable.LESS_THAN_OR_EQUAL; + case GREATER_THAN_OR_EQUAL: + return SqlStdOperatorTable.GREATER_THAN_OR_EQUAL; + default: + throw new AssertionError(kind); + } + } + + public static class ExprSimplifier extends RexShuttle { + private final RexBuilder rexBuilder; + private final Map unknownAsFalseMap; + + public ExprSimplifier(RexBuilder rexBuilder) { + this.rexBuilder = rexBuilder; + this.unknownAsFalseMap = new HashMap<>(); + } + + @Override + public RexNode visitCall(RexCall call) { + Boolean unknownAsFalse; + switch (call.getKind()) { + case AND: + case CASE: + unknownAsFalse = this.unknownAsFalseMap.get(call); + if (unknownAsFalse == null) { + // Top operator + unknownAsFalse = true; + } + break; + default: + unknownAsFalse = false; + } + for (RexNode operand : call.operands) { + this.unknownAsFalseMap.put(operand, unknownAsFalse); + } + RexNode node = super.visitCall(call); + RexNode simplifiedNode = visit(node, unknownAsFalse); + if (simplifiedNode != null) { + return simplifiedNode; + } + return node; + } + + private RexNode visit(final RexNode call, boolean unknownAsFalse) { + RexNode simplified = HiveRexUtil.simplify(rexBuilder, call, unknownAsFalse); + if (simplified == null) { + return null; + } + if (simplified.getType().equals(call.getType())) { + return simplified; + } + return rexBuilder.makeCast(call.getType(), simplified, true); + } + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregatePullUpConstantsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregatePullUpConstantsRule.java new file mode 100644 index 0000000..370c0ec --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregatePullUpConstantsRule.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.rules.AggregateProjectPullUpConstantsRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; + +public class HiveAggregatePullUpConstantsRule extends AggregateProjectPullUpConstantsRule { + + public static final HiveAggregatePullUpConstantsRule INSTANCE = + new HiveAggregatePullUpConstantsRule(); + + public HiveAggregatePullUpConstantsRule() { + super(HiveAggregate.class, RelNode.class, + HiveRelFactories.HIVE_BUILDER, "HiveAggregatePullUpConstantsRule"); + } + + @Override + public boolean matches(RelOptRuleCall call) { + final Aggregate aggregate = call.rel(0); + // Rule cannot be applied if there are GroupingSets + if (aggregate.indicator) { + return false; + } + return super.matches(call); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectFilterPullUpConstantsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectFilterPullUpConstantsRule.java new file mode 100644 index 0000000..defab09 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectFilterPullUpConstantsRule.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +/** + * Planner rule that infers constant expressions from Filter into + * a Project operator. + */ +public class HiveProjectFilterPullUpConstantsRule extends RelOptRule { + + protected static final Logger LOG = LoggerFactory.getLogger( + HiveProjectFilterPullUpConstantsRule.class); + + public static final HiveProjectFilterPullUpConstantsRule INSTANCE = + new HiveProjectFilterPullUpConstantsRule(HiveProject.class, HiveFilter.class, + HiveRelFactories.HIVE_BUILDER); + + public HiveProjectFilterPullUpConstantsRule( + Class projectClass, + Class filterClass, + RelBuilderFactory relBuilderFactory) { + super(operand(projectClass, + operand(filterClass, any())), + relBuilderFactory, null); + } + + @Override + public boolean matches(RelOptRuleCall call) { + final Filter filterRel = call.rel(1); + RexNode condition = filterRel.getCondition(); + if (!HiveCalciteUtil.isDeterministic(condition)) { + return false; + } + + return super.matches(call); + } + + public void onMatch(RelOptRuleCall call) { + final Project project = call.rel(0); + final Filter filter = call.rel(1); + final RelBuilder builder = call.builder(); + + List projects = project.getChildExps(); + List newProjects = rewriteProjects(projects, filter.getCondition(), builder); + if (newProjects == null) { + return; + } + + RelNode newProjRel = builder.push(filter) + .project(newProjects, project.getRowType().getFieldNames()).build(); + call.transformTo(newProjRel); + } + + // Rewrite projects to replace column references by constants when possible + @SuppressWarnings("incomplete-switch") + private static List rewriteProjects(List projects, + RexNode newPushedCondition, RelBuilder relBuilder) { + final List conjunctions = RelOptUtil.conjunctions(newPushedCondition); + final Map conditions = new HashMap(); + for (RexNode conjunction: conjunctions) { + // 1.1. If it is not a RexCall, we continue + if (!(conjunction instanceof RexCall)) { + continue; + } + // 1.2. We extract the information that we need + RexCall conjCall = (RexCall) conjunction; + switch (conjCall.getOperator().getKind()) { + case EQUALS: + if (!(RexUtil.isConstant(conjCall.operands.get(0))) && + RexUtil.isConstant(conjCall.operands.get(1))) { + conditions.put(conjCall.operands.get(0).toString(), conjCall.operands.get(1)); + } else if (!(RexUtil.isConstant(conjCall.operands.get(1))) && + RexUtil.isConstant(conjCall.operands.get(0))) { + conditions.put(conjCall.operands.get(1).toString(), conjCall.operands.get(0)); + } + break; + case IS_NULL: + conditions.put(conjCall.operands.get(0).toString(), + relBuilder.getRexBuilder().makeNullLiteral( + conjCall.operands.get(0).getType().getSqlTypeName())); + } + } + + RexReplacer replacer = new RexReplacer(relBuilder.getRexBuilder(), conditions); + List newProjects = Lists.newArrayList(projects); + replacer.mutate(newProjects); + if (replacer.replaced) { + return newProjects; + } + return null; + } + + protected static class RexReplacer extends RexShuttle { + private final RexBuilder rexBuilder; + private final Map replacements; + private boolean replaced; + + RexReplacer( + RexBuilder rexBuilder, + Map replacements) { + this.rexBuilder = rexBuilder; + this.replacements = replacements; + this.replaced = false; + } + + @Override public RexNode visitInputRef(RexInputRef inputRef) { + RexNode node = visit(inputRef); + if (node == null) { + return super.visitInputRef(inputRef); + } + this.replaced = true; + return node; + } + + @Override public RexNode visitCall(RexCall call) { + RexNode node = visit(call); + if (node != null) { + this.replaced = true; + return node; + } + return super.visitCall(call); + } + + private RexNode visit(final RexNode call) { + RexNode replacement = replacements.get(call.toString()); + if (replacement == null) { + return null; + } + if (replacement.getType().equals(call.getType())) { + return replacement; + } + return rexBuilder.makeCast(call.getType(), replacement, true); + } + } +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java index 2fe9b75..f04e13f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java @@ -59,11 +59,15 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.Stacks; import org.apache.calcite.util.Util; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexUtil; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexUtil.ExprSimplifier; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -80,6 +84,9 @@ * */ public abstract class HiveReduceExpressionsRule extends RelOptRule { + + protected static final Logger LOG = LoggerFactory.getLogger(HiveReduceExpressionsRule.class); + //~ Static fields/initializers --------------------------------------------- /** @@ -125,17 +132,23 @@ public FilterReduceExpressionsRule(Class filterClass, @Override public void onMatch(RelOptRuleCall call) { final Filter filter = call.rel(0); - final RexBuilder rexBuilder = filter.getCluster().getRexBuilder(); - - RexNode newConditionExp = HiveRexUtil.simplify(rexBuilder, filter.getCondition()); - final List expList = Lists.newArrayList(newConditionExp); - boolean reduced = false; + final List expList = + Lists.newArrayList(filter.getCondition()); + RexNode newConditionExp; + boolean reduced; final RelOptPredicateList predicates = RelMetadataQuery.instance().getPulledUpPredicates(filter.getInput()); - if (reduceExpressions(filter, expList, predicates)) { + if (reduceExpressions(filter, expList, predicates, true)) { assert expList.size() == 1; newConditionExp = expList.get(0); reduced = true; + } else { + // No reduction, but let's still test the original + // predicate to see if it was already a constant, + // in which case we don't need any runtime decision + // about filtering. + newConditionExp = filter.getCondition(); + reduced = false; } // Even if no reduction, let's still test the original @@ -146,8 +159,7 @@ public FilterReduceExpressionsRule(Class filterClass, if (newConditionExp.isAlwaysTrue()) { call.transformTo( filter.getInput()); - } else if (reduced - || !newConditionExp.toString().equals(filter.getCondition().toString())) { + } else if (reduced) { call.transformTo(call.builder(). push(filter.getInput()).filter(newConditionExp).build()); } else { @@ -169,26 +181,8 @@ public ProjectReduceExpressionsRule(Class projectClass, super(projectClass, relBuilderFactory, "HiveReduceExpressionsRule(Project)"); } - public boolean matches(RelOptRuleCall call) { - Project project = call.rel(0); - HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); - - // If this operator has been visited already by the rule, - // we do not need to apply the optimization - if (registry != null && registry.getVisited(this).contains(project)) { - return false; - } - - return true; - } - @Override public void onMatch(RelOptRuleCall call) { Project project = call.rel(0); - // Register that we have visited this operator in this rule - HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); - if (registry != null) { - registry.registerVisited(this, project); - } final RelOptPredicateList predicates = RelMetadataQuery.instance().getPulledUpPredicates(project.getInput()); final List expList = @@ -196,9 +190,6 @@ public boolean matches(RelOptRuleCall call) { if (reduceExpressions(project, expList, predicates)) { RelNode newProject = call.builder().push(project.getInput()) .project(expList, project.getRowType().getFieldNames()).build(); - if (registry != null) { - registry.registerVisited(this, newProject); - } call.transformTo(newProject); // New plan is absolutely better than old plan. @@ -275,6 +266,43 @@ protected HiveReduceExpressionsRule(Class clazz, */ protected static boolean reduceExpressions(RelNode rel, List expList, RelOptPredicateList predicates) { + return reduceExpressions(rel, expList, predicates, false); + } + + /** + * Reduces a list of expressions. + * + * @param rel Relational expression + * @param expList List of expressions, modified in place + * @param predicates Constraints known to hold on input expressions + * @param unknownAsFalse Whether UNKNOWN will be treated as FALSE + * + * @return whether reduction found something to change, and succeeded + */ + protected static boolean reduceExpressions(RelNode rel, List expList, + RelOptPredicateList predicates, boolean unknownAsFalse) { + RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); + + boolean reduced = reduceExpressionsInternal(rel, expList, predicates); + + // Simplify preds in place + ExprSimplifier simplifier = new ExprSimplifier(rexBuilder); + List expList2 = Lists.newArrayList(expList); + simplifier.mutate(expList2); + boolean simplified = false; + for (int i = 0; i < expList.size(); i++) { + if (!expList2.get(i).toString().equals(expList.get(i).toString())) { + expList.remove(i); + expList.add(i, expList2.get(i)); + simplified = true; + } + } + + return reduced || simplified; + } + + protected static boolean reduceExpressionsInternal(RelNode rel, List expList, + RelOptPredicateList predicates) { RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); // Replace predicates on CASE to CASE on predicates. @@ -284,8 +312,8 @@ protected static boolean reduceExpressions(RelNode rel, List expList, final List constExps = Lists.newArrayList(); List addCasts = Lists.newArrayList(); final List removableCasts = Lists.newArrayList(); - final ImmutableMap constants = - predicateConstants(predicates); + final ImmutableMap constants = + predicateConstants(RexNode.class, rexBuilder, predicates); findReducibleExps(rel.getCluster().getTypeFactory(), expList, constants, constExps, addCasts, removableCasts); if (constExps.isEmpty() && removableCasts.isEmpty()) { @@ -347,6 +375,13 @@ protected static boolean reduceExpressions(RelNode rel, List expList, final List reducedValues = Lists.newArrayList(); executor.reduce(rexBuilder, constExps2, reducedValues); + // Use RexNode.digest to judge whether each newly generated RexNode + // is equivalent to the original one. + if (Lists.transform(constExps, HiveCalciteUtil.REX_STR_FN).equals( + Lists.transform(reducedValues, HiveCalciteUtil.REX_STR_FN))) { + return false; + } + // For Project, we have to be sure to preserve the result // types, so always cast regardless of the expression type. // For other RelNodes like Filter, in general, this isn't necessary, @@ -384,7 +419,7 @@ protected static boolean reduceExpressions(RelNode rel, List expList, * @param removableCasts returns the list of cast expressions where the cast */ protected static void findReducibleExps(RelDataTypeFactory typeFactory, - List exps, ImmutableMap constants, + List exps, ImmutableMap constants, List constExps, List addCasts, List removableCasts) { ReducibleExprLocator gardener = @@ -437,18 +472,26 @@ protected static void findReducibleExps(RelDataTypeFactory typeFactory, private static void gatherConstraints(Class clazz, RexNode predicate, Map map, Set excludeSet, RexBuilder rexBuilder) { - if (predicate.getKind() != SqlKind.EQUALS) { + if (predicate.getKind() != SqlKind.EQUALS + && predicate.getKind() != SqlKind.IS_NULL) { decompose(excludeSet, predicate); return; } final List operands = ((RexCall) predicate).getOperands(); - if (operands.size() != 2) { + if (operands.size() != 2 && predicate.getKind() == SqlKind.EQUALS) { decompose(excludeSet, predicate); return; } // if it reaches here, we have rexNode equals rexNode - final RexNode left = operands.get(0); - final RexNode right = operands.get(1); + final RexNode left; + final RexNode right; + if (predicate.getKind() == SqlKind.EQUALS) { + left = operands.get(0); + right = operands.get(1); + } else { + left = operands.get(0); + right = rexBuilder.makeNullLiteral(left.getType().getSqlTypeName()); + } // note that literals are immutable too and they can only be compared through // values. gatherConstraint(clazz, left, right, map, excludeSet, rexBuilder); @@ -520,33 +563,6 @@ private static boolean canAssignFrom(RelDataType type1, RelDataType type2) { } } } - - protected static ImmutableMap predicateConstants( - RelOptPredicateList predicates) { - // We cannot use an ImmutableMap.Builder here. If there are multiple entries - // with the same key (e.g. "WHERE deptno = 1 AND deptno = 2"), it doesn't - // matter which we take, so the latter will replace the former. - // The basic idea is to find all the pairs of RexNode = RexLiteral - // (1) If 'predicates' contain a non-EQUALS, we bail out. - // (2) It is OK if a RexNode is equal to the same RexLiteral several times, - // (e.g. "WHERE deptno = 1 AND deptno = 1") - // (3) It will return false if there are inconsistent constraints (e.g. - // "WHERE deptno = 1 AND deptno = 2") - final Map map = new HashMap<>(); - final Set excludeSet = new HashSet<>(); - for (RexNode predicate : predicates.pulledUpPredicates) { - gatherConstraints(map, predicate, excludeSet); - } - final ImmutableMap.Builder builder = - ImmutableMap.builder(); - for (Map.Entry entry : map.entrySet()) { - RexNode rexNode = entry.getKey(); - if (!overlap(rexNode, excludeSet)) { - builder.put(rexNode, entry.getValue()); - } - } - return builder.build(); - } private static boolean overlap(RexNode rexNode, Set set) { if (rexNode instanceof RexCall) { @@ -573,46 +589,6 @@ private static void decompose(Set set, RexNode rexNode) { } } - private static void gatherConstraints(Map map, - RexNode predicate, Set excludeSet) { - if (predicate.getKind() != SqlKind.EQUALS) { - decompose(excludeSet, predicate); - return; - } - final List operands = ((RexCall) predicate).getOperands(); - if (operands.size() != 2) { - decompose(excludeSet, predicate); - return; - } - // if it reaches here, we have rexNode equals rexNode - final RexNode left = operands.get(0); - final RexNode right = operands.get(1); - // note that literals are immutable too and they can only be compared through - // values. - if (right instanceof RexLiteral && !excludeSet.contains(left)) { - RexLiteral existedValue = map.get(left); - if (existedValue == null) { - map.put(left, (RexLiteral) right); - } else { - if (!existedValue.getValue().equals(((RexLiteral) right).getValue())) { - // we found conflict values. - map.remove(left); - excludeSet.add(left); - } - } - } else if (left instanceof RexLiteral && !excludeSet.contains(right)) { - RexLiteral existedValue = map.get(right); - if (existedValue == null) { - map.put(right, (RexLiteral) left); - } else { - if (!existedValue.getValue().equals(((RexLiteral) left).getValue())) { - map.remove(right); - excludeSet.add(right); - } - } - } - } - /** Pushes predicates into a CASE. * *

We have a loose definition of 'predicate': any boolean expression will @@ -743,7 +719,7 @@ private RexNode visit(final RexNode call) { private final List stack; - private final ImmutableMap constants; + private final ImmutableMap constants; private final List constExprs; @@ -754,7 +730,7 @@ private RexNode visit(final RexNode call) { private final List parentCallTypeStack; ReducibleExprLocator(RelDataTypeFactory typeFactory, - ImmutableMap constants, List constExprs, + ImmutableMap constants, List constExprs, List addCasts, List removableCasts) { // go deep super(true); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortLimitPullUpConstantsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortLimitPullUpConstantsRule.java index 3be9b0a..ebfabac 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortLimitPullUpConstantsRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortLimitPullUpConstantsRule.java @@ -92,8 +92,15 @@ public void onMatch(RelOptRuleCall call) { return; } - Map constants = HiveReduceExpressionsRule.predicateConstants( + Map conditionsExtracted = HiveReduceExpressionsRule.predicateConstants( RexNode.class, rexBuilder, predicates); + Map constants = new HashMap<>(); + for (int i = 0; i < count ; i++) { + RexNode expr = rexBuilder.makeInputRef(sort.getInput(), i); + if (conditionsExtracted.containsKey(expr)) { + constants.put(expr, conditionsExtracted.get(expr)); + } + } // None of the expressions are constant. Nothing to do. if (constants.isEmpty()) { @@ -102,13 +109,10 @@ public void onMatch(RelOptRuleCall call) { if (count == constants.size()) { // At least a single item in project is required. - final Map map = new HashMap<>(constants); - map.remove(map.keySet().iterator().next()); - constants = map; + constants.remove(constants.keySet().iterator().next()); } // Create expressions for Project operators before and after the Sort - boolean atLeastOneConstant = false; List fields = sort.getInput().getRowType().getFieldList(); List> newChildExprs = new ArrayList<>(); List topChildExprs = new ArrayList<>(); @@ -117,7 +121,6 @@ public void onMatch(RelOptRuleCall call) { RexNode expr = rexBuilder.makeInputRef(sort.getInput(), i); RelDataTypeField field = fields.get(i); if (constants.containsKey(expr)) { - atLeastOneConstant = true; topChildExprs.add(constants.get(expr)); topChildExprsFields.add(field.getName()); } else { @@ -127,11 +130,6 @@ public void onMatch(RelOptRuleCall call) { } } - // No constants were found - if (!atLeastOneConstant) { - return; - } - // Update field collations final Mappings.TargetMapping mapping = RelOptUtil.permutation(Pair.left(newChildExprs), sort.getInput().getRowType()).inverse(); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java index 2552f87..f071ddd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveUnionPullUpConstantsRule.java @@ -80,32 +80,30 @@ public void onMatch(RelOptRuleCall call) { return; } - Map constants = HiveReduceExpressionsRule.predicateConstants( + Map conditionsExtracted = HiveReduceExpressionsRule.predicateConstants( RexNode.class, rexBuilder, predicates); + Map constants = new HashMap<>(); + for (int i = 0; i < count ; i++) { + RexNode expr = rexBuilder.makeInputRef(union, i); + if (conditionsExtracted.containsKey(expr)) { + constants.put(expr, conditionsExtracted.get(expr)); + } + } // None of the expressions are constant. Nothing to do. if (constants.isEmpty()) { return; } - if (count == constants.size()) { - // At least a single item in project is required. - final Map map = new HashMap<>(constants); - map.remove(map.keySet().iterator().next()); - constants = map; - } - // Create expressions for Project operators before and after the Union - boolean atLeastOneConstant = false; - List fields = union.getRowType().getFieldList(); List> newChildExprs = new ArrayList<>(); + List fields = union.getRowType().getFieldList(); List topChildExprs = new ArrayList<>(); List topChildExprsFields = new ArrayList<>(); for (int i = 0; i < count ; i++) { RexNode expr = rexBuilder.makeInputRef(union, i); RelDataTypeField field = fields.get(i); if (constants.containsKey(expr)) { - atLeastOneConstant = true; topChildExprs.add(constants.get(expr)); topChildExprsFields.add(field.getName()); } else { @@ -115,9 +113,10 @@ public void onMatch(RelOptRuleCall call) { } } - // No constants were found - if (!atLeastOneConstant) { - return; + if (newChildExprs.isEmpty()) { + // At least a single item in project is required. + newChildExprs.add(Pair.of( + topChildExprs.get(0), topChildExprsFields.get(0))); } // Update top Project positions diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java index e810747..85e66d5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java @@ -18,44 +18,24 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.stats; import java.util.ArrayList; -import java.util.BitSet; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import org.apache.calcite.linq4j.Linq4j; import org.apache.calcite.linq4j.Ord; -import org.apache.calcite.linq4j.function.Predicate1; -import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdPredicates; import org.apache.calcite.rel.metadata.RelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexPermutationShuttle; import org.apache.calcite.rex.RexPermuteInputsShuttle; -import org.apache.calcite.rex.RexShuttle; -import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.rex.RexVisitorImpl; -import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.util.BitSets; import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.mapping.Mapping; @@ -63,13 +43,8 @@ import org.apache.calcite.util.mapping.Mappings; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; -import com.google.common.base.Function; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; //TODO: Move this to calcite @@ -151,511 +126,4 @@ public RelOptPredicateList getPredicates(Project project, RelMetadataQuery mq) { return RelOptPredicateList.of(projectPullUpPredicates); } - /** Infers predicates for a {@link org.apache.calcite.rel.core.Join}. */ - @Override - public RelOptPredicateList getPredicates(Join join, RelMetadataQuery mq) { - RexBuilder rB = join.getCluster().getRexBuilder(); - RelNode left = join.getInput(0); - RelNode right = join.getInput(1); - - RelOptPredicateList leftInfo = mq.getPulledUpPredicates(left); - RelOptPredicateList rightInfo = mq.getPulledUpPredicates(right); - - HiveJoinConditionBasedPredicateInference jI = new HiveJoinConditionBasedPredicateInference(join, - RexUtil.composeConjunction(rB, leftInfo.pulledUpPredicates, false), - RexUtil.composeConjunction(rB, rightInfo.pulledUpPredicates, false)); - - return jI.inferPredicates(false); - } - - /** - * Utility to infer predicates from one side of the join that apply on the - * other side. - * - *

Contract is:

    - * - *
  • initialize with a {@link org.apache.calcite.rel.core.Join} and - * optional predicates applicable on its left and right subtrees. - * - *
  • you can - * then ask it for equivalentPredicate(s) given a predicate. - * - *
- * - *

So for: - *

    - *
  1. 'R1(x) join R2(y) on x = y' a call for - * equivalentPredicates on 'x > 7' will return ' - * [y > 7]' - *
  2. 'R1(x) join R2(y) on x = y join R3(z) on y = z' a call for - * equivalentPredicates on the second join 'x > 7' will return ' - * [y > 7, z > 7]' - *
- */ - static class HiveJoinConditionBasedPredicateInference { - final Join joinRel; - final boolean isSemiJoin; - final int nSysFields; - final int nFieldsLeft; - final int nFieldsRight; - final ImmutableBitSet leftFieldsBitSet; - final ImmutableBitSet rightFieldsBitSet; - final ImmutableBitSet allFieldsBitSet; - SortedMap equivalence; - final Map exprFields; - final Set allExprsDigests; - final Set equalityPredicates; - final RexNode leftChildPredicates; - final RexNode rightChildPredicates; - - public HiveJoinConditionBasedPredicateInference(Join joinRel, - RexNode lPreds, RexNode rPreds) { - this(joinRel, joinRel instanceof SemiJoin, lPreds, rPreds); - } - - private HiveJoinConditionBasedPredicateInference(Join joinRel, boolean isSemiJoin, - RexNode lPreds, RexNode rPreds) { - super(); - this.joinRel = joinRel; - this.isSemiJoin = isSemiJoin; - nFieldsLeft = joinRel.getLeft().getRowType().getFieldList().size(); - nFieldsRight = joinRel.getRight().getRowType().getFieldList().size(); - nSysFields = joinRel.getSystemFieldList().size(); - leftFieldsBitSet = ImmutableBitSet.range(nSysFields, - nSysFields + nFieldsLeft); - rightFieldsBitSet = ImmutableBitSet.range(nSysFields + nFieldsLeft, - nSysFields + nFieldsLeft + nFieldsRight); - allFieldsBitSet = ImmutableBitSet.range(0, - nSysFields + nFieldsLeft + nFieldsRight); - - exprFields = Maps.newHashMap(); - allExprsDigests = new HashSet(); - - if (lPreds == null) { - leftChildPredicates = null; - } else { - Mappings.TargetMapping leftMapping = Mappings.createShiftMapping( - nSysFields + nFieldsLeft, nSysFields, 0, nFieldsLeft); - leftChildPredicates = lPreds.accept( - new RexPermuteInputsShuttle(leftMapping, joinRel.getInput(0))); - - for (RexNode r : RelOptUtil.conjunctions(leftChildPredicates)) { - exprFields.put(r.toString(), RelOptUtil.InputFinder.bits(r)); - allExprsDigests.add(r.toString()); - } - } - if (rPreds == null) { - rightChildPredicates = null; - } else { - Mappings.TargetMapping rightMapping = Mappings.createShiftMapping( - nSysFields + nFieldsLeft + nFieldsRight, - nSysFields + nFieldsLeft, 0, nFieldsRight); - rightChildPredicates = rPreds.accept( - new RexPermuteInputsShuttle(rightMapping, joinRel.getInput(1))); - - for (RexNode r : RelOptUtil.conjunctions(rightChildPredicates)) { - exprFields.put(r.toString(), RelOptUtil.InputFinder.bits(r)); - allExprsDigests.add(r.toString()); - } - } - - equivalence = Maps.newTreeMap(); - equalityPredicates = new HashSet(); - for (int i = 0; i < nSysFields + nFieldsLeft + nFieldsRight; i++) { - equivalence.put(i, BitSets.of(i)); - } - - // Only process equivalences found in the join conditions. Processing - // Equivalences from the left or right side infer predicates that are - // already present in the Tree below the join. - RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder(); - List exprs = - RelOptUtil.conjunctions( - compose(rexBuilder, ImmutableList.of(joinRel.getCondition()))); - - final EquivalenceFinder eF = new EquivalenceFinder(); - new ArrayList(Lists.transform(exprs, new Function() { - public Void apply(RexNode input) { - return input.accept(eF); - } - })); - - equivalence = BitSets.closure(equivalence); - } - - /** - * The PullUp Strategy is sound but not complete. - *
    - *
  1. We only pullUp inferred predicates for now. Pulling up existing - * predicates causes an explosion of duplicates. The existing predicates are - * pushed back down as new predicates. Once we have rules to eliminate - * duplicate Filter conditions, we should pullUp all predicates. - *
  2. For Left Outer: we infer new predicates from the left and set them as - * applicable on the Right side. No predicates are pulledUp. - *
  3. Right Outer Joins are handled in an analogous manner. - *
  4. For Full Outer Joins no predicates are pulledUp or inferred. - *
- */ - public RelOptPredicateList inferPredicates( - boolean includeEqualityInference) { - List inferredPredicates = new ArrayList(); - Set allExprsDigests = new HashSet(this.allExprsDigests); - final JoinRelType joinType = joinRel.getJoinType(); - switch (joinType) { - case INNER: - case LEFT: - infer(leftChildPredicates, allExprsDigests, inferredPredicates, - includeEqualityInference, - joinType == JoinRelType.LEFT ? rightFieldsBitSet - : allFieldsBitSet); - break; - } - switch (joinType) { - case INNER: - case RIGHT: - infer(rightChildPredicates, allExprsDigests, inferredPredicates, - includeEqualityInference, - joinType == JoinRelType.RIGHT ? leftFieldsBitSet - : allFieldsBitSet); - break; - } - - Mappings.TargetMapping rightMapping = Mappings.createShiftMapping( - nSysFields + nFieldsLeft + nFieldsRight, - 0, nSysFields + nFieldsLeft, nFieldsRight); - final HiveJoinRexPermuteInputsShuttle rightPermute = - new HiveJoinRexPermuteInputsShuttle(rightMapping, joinRel); - Mappings.TargetMapping leftMapping = Mappings.createShiftMapping( - nSysFields + nFieldsLeft, 0, nSysFields, nFieldsLeft); - final HiveJoinRexPermuteInputsShuttle leftPermute = - new HiveJoinRexPermuteInputsShuttle(leftMapping, joinRel); - - List leftInferredPredicates = new ArrayList(); - List rightInferredPredicates = new ArrayList(); - - for (RexNode iP : inferredPredicates) { - ImmutableBitSet iPBitSet = RelOptUtil.InputFinder.bits(iP); - if (iPBitSet.isEmpty() && joinType == JoinRelType.INNER) { - leftInferredPredicates.add(iP); - rightInferredPredicates.add(iP); - } else if (iPBitSet.isEmpty() && joinType == JoinRelType.LEFT) { - rightInferredPredicates.add(iP); - } else if (iPBitSet.isEmpty() && joinType == JoinRelType.RIGHT) { - leftInferredPredicates.add(iP); - } else if (leftFieldsBitSet.contains(iPBitSet)) { - leftInferredPredicates.add(iP.accept(leftPermute)); - } else if (rightFieldsBitSet.contains(iPBitSet)) { - rightInferredPredicates.add(iP.accept(rightPermute)); - } - } - - switch (joinType) { - case INNER: - Iterable pulledUpPredicates; - if (isSemiJoin) { - pulledUpPredicates = Iterables.concat( - RelOptUtil.conjunctions(leftChildPredicates), - leftInferredPredicates); - } else { - pulledUpPredicates = Iterables.concat( - RelOptUtil.conjunctions(leftChildPredicates), - RelOptUtil.conjunctions(rightChildPredicates), - RelOptUtil.conjunctions(joinRel.getCondition()), - inferredPredicates); - } - return RelOptPredicateList.of(pulledUpPredicates, - leftInferredPredicates, rightInferredPredicates); - case LEFT: - return RelOptPredicateList.of( - RelOptUtil.conjunctions(leftChildPredicates), - EMPTY_LIST, rightInferredPredicates); - case RIGHT: - return RelOptPredicateList.of( - RelOptUtil.conjunctions(rightChildPredicates), - leftInferredPredicates, EMPTY_LIST); - default: - assert inferredPredicates.size() == 0; - return RelOptPredicateList.EMPTY; - } - } - - public RexNode left() { - return leftChildPredicates; - } - - public RexNode right() { - return rightChildPredicates; - } - - private void infer(RexNode predicates, Set allExprsDigests, - List inferedPredicates, boolean includeEqualityInference, - ImmutableBitSet inferringFields) { - for (RexNode r : RelOptUtil.conjunctions(predicates)) { - if (r.isAlwaysFalse()) { - RexLiteral falseVal = - joinRel.getCluster().getRexBuilder().makeLiteral(false); - inferedPredicates.add(falseVal); - allExprsDigests.add(falseVal.toString()); - continue; - } - if (!includeEqualityInference - && equalityPredicates.contains(r.toString())) { - continue; - } - for (Mapping m : mappings(r)) { - RexNode tr = r.accept( - new RexPermuteInputsShuttle(m, joinRel.getInput(0), - joinRel.getInput(1))); - if (inferringFields.contains(RelOptUtil.InputFinder.bits(tr)) - && !allExprsDigests.contains(tr.toString()) - && !isAlwaysTrue(tr)) { - inferedPredicates.add(tr); - allExprsDigests.add(tr.toString()); - } - } - } - } - - Iterable mappings(final RexNode predicate) { - return new Iterable() { - public Iterator iterator() { - ImmutableBitSet fields = exprFields.get(predicate.toString()); - if (fields.cardinality() == 0) { - return Iterators.emptyIterator(); - } - return new ExprsItr(fields); - } - }; - } - - private void equivalent(int p1, int p2) { - BitSet b = equivalence.get(p1); - b.set(p2); - - b = equivalence.get(p2); - b.set(p1); - } - - RexNode compose(RexBuilder rexBuilder, Iterable exprs) { - exprs = Linq4j.asEnumerable(exprs).where(new Predicate1() { - public boolean apply(RexNode expr) { - return expr != null; - } - }); - return RexUtil.composeConjunction(rexBuilder, exprs, false); - } - - /** - * Find expressions of the form 'col_x = col_y'. - */ - class EquivalenceFinder extends RexVisitorImpl { - protected EquivalenceFinder() { - super(true); - } - - @Override public Void visitCall(RexCall call) { - if (call.getOperator().getKind() == SqlKind.EQUALS) { - int lPos = pos(call.getOperands().get(0)); - int rPos = pos(call.getOperands().get(1)); - if (lPos != -1 && rPos != -1) { - HiveJoinConditionBasedPredicateInference.this.equivalent(lPos, rPos); - HiveJoinConditionBasedPredicateInference.this.equalityPredicates - .add(call.toString()); - } - } - return null; - } - } - - /** - * Given an expression returns all the possible substitutions. - * - *

For example, for an expression 'a + b + c' and the following - * equivalences:

-     * a : {a, b}
-     * b : {a, b}
-     * c : {c, e}
-     * 
- * - *

The following Mappings will be returned: - *

-     * {a->a, b->a, c->c}
-     * {a->a, b->a, c->e}
-     * {a->a, b->b, c->c}
-     * {a->a, b->b, c->e}
-     * {a->b, b->a, c->c}
-     * {a->b, b->a, c->e}
-     * {a->b, b->b, c->c}
-     * {a->b, b->b, c->e}
-     * 
- * - *

which imply the following inferences: - *

-     * a + a + c
-     * a + a + e
-     * a + b + c
-     * a + b + e
-     * b + a + c
-     * b + a + e
-     * b + b + c
-     * b + b + e
-     * 
- */ - class ExprsItr implements Iterator { - final int[] columns; - final BitSet[] columnSets; - final int[] iterationIdx; - Mapping nextMapping; - boolean firstCall; - - ExprsItr(ImmutableBitSet fields) { - nextMapping = null; - columns = new int[fields.cardinality()]; - columnSets = new BitSet[fields.cardinality()]; - iterationIdx = new int[fields.cardinality()]; - for (int j = 0, i = fields.nextSetBit(0); i >= 0; i = fields - .nextSetBit(i + 1), j++) { - columns[j] = i; - columnSets[j] = equivalence.get(i); - iterationIdx[j] = 0; - } - firstCall = true; - } - - public boolean hasNext() { - if (firstCall) { - initializeMapping(); - firstCall = false; - } else { - computeNextMapping(iterationIdx.length - 1); - } - return nextMapping != null; - } - - public Mapping next() { - return nextMapping; - } - - public void remove() { - throw new UnsupportedOperationException(); - } - - private void computeNextMapping(int level) { - int t = columnSets[level].nextSetBit(iterationIdx[level]); - if (t < 0) { - if (level == 0) { - nextMapping = null; - } else { - iterationIdx[level] = 0; - computeNextMapping(level - 1); - } - } else { - nextMapping.set(columns[level], t); - iterationIdx[level] = t + 1; - } - } - - private void initializeMapping() { - nextMapping = Mappings.create(MappingType.PARTIAL_FUNCTION, - nSysFields + nFieldsLeft + nFieldsRight, - nSysFields + nFieldsLeft + nFieldsRight); - for (int i = 0; i < columnSets.length; i++) { - BitSet c = columnSets[i]; - int t = c.nextSetBit(iterationIdx[i]); - if (t < 0) { - nextMapping = null; - return; - } - nextMapping.set(columns[i], t); - iterationIdx[i] = t + 1; - } - } - } - - private int pos(RexNode expr) { - if (expr instanceof RexInputRef) { - return ((RexInputRef) expr).getIndex(); - } - return -1; - } - - private boolean isAlwaysTrue(RexNode predicate) { - if (predicate instanceof RexCall) { - RexCall c = (RexCall) predicate; - if (c.getOperator().getKind() == SqlKind.EQUALS) { - int lPos = pos(c.getOperands().get(0)); - int rPos = pos(c.getOperands().get(1)); - return lPos != -1 && lPos == rPos; - } - } - return predicate.isAlwaysTrue(); - } - } - - /** - * Shuttle which applies a permutation to its input fields. - * - * @see RexPermutationShuttle - * @see RexUtil#apply(org.apache.calcite.util.mapping.Mappings.TargetMapping, RexNode) - */ - public static class HiveJoinRexPermuteInputsShuttle extends RexShuttle { - //~ Instance fields -------------------------------------------------------- - - private final Mappings.TargetMapping mapping; - private final ImmutableList fields; - private final RelOptCluster cluster; - private final RelDataType rType; - - //~ Constructors ----------------------------------------------------------- - - private HiveJoinRexPermuteInputsShuttle( - Mappings.TargetMapping mapping, - RelNode input) { - this.mapping = mapping; - this.cluster = input.getCluster(); - this.rType = input.getRowType(); - this.fields = ImmutableList.copyOf(rType.getFieldList()); - } - - //~ Methods ---------------------------------------------------------------- - - private static ImmutableList fields(RelNode[] inputs) { - final ImmutableList.Builder fields = - ImmutableList.builder(); - for (RelNode input : inputs) { - fields.addAll(input.getRowType().getFieldList()); - } - return fields.build(); - } - - @Override public RexNode visitInputRef(RexInputRef local) { - final int index = local.getIndex(); - int target = mapping.getTarget(index); - return new RexInputRef( - target, - fields.get(index).getType()); - } - - @Override public RexNode visitCall(RexCall call) { - if (call.getOperator() == RexBuilder.GET_OPERATOR) { - final String name = - (String) ((RexLiteral) call.getOperands().get(1)).getValue2(); - final int i = lookup(fields, name); - if (i >= 0) { - return RexInputRef.of(i, fields); - } - } - return HiveCalciteUtil.getTypeSafePred(cluster, super.visitCall(call), rType); - } - - private static int lookup(List fields, String name) { - for (int i = 0; i < fields.size(); i++) { - final RelDataTypeField field = fields.get(i); - if (field.getName().equals(name)) { - return i; - } - } - return -1; - } - } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java index de7e2f8..353d8db 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java @@ -45,6 +45,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.rex.RexWindow; import org.apache.calcite.rex.RexWindowBound; @@ -189,6 +190,10 @@ else if (aggregateType == Group.CUBE) { int i = 0; for (RexNode r : select.getChildExps()) { + if (RexUtil.isNull(r) && r.getType().getSqlTypeName() != SqlTypeName.NULL) { + // It is NULL value with different type, we need to introduce a CAST to keep it + r = select.getCluster().getRexBuilder().makeAbstractCast(r.getType(), r); + } ASTNode expr = r.accept(new RexVisitor(schema, r instanceof RexLiteral)); String alias = select.getRowType().getFieldNames().get(i++); ASTNode selectExpr = ASTBuilder.selectExpr(expr, alias); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java index e51b6c4..92c28f3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java @@ -36,6 +36,7 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.rex.RexWindow; import org.apache.calcite.rex.RexWindowBound; @@ -45,6 +46,7 @@ import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.RexVisitor; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.Schema; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -79,6 +81,7 @@ */ public class ExprNodeConverter extends RexVisitorImpl { + private final boolean foldExpr; private final String tabAlias; private final RelDataType inputRowType; private final ImmutableSet inputVCols; @@ -89,16 +92,28 @@ public ExprNodeConverter(String tabAlias, RelDataType inputRowType, Set vCols, RelDataTypeFactory dTFactory) { - this(tabAlias, null, inputRowType, null, vCols, dTFactory); + this(tabAlias, null, inputRowType, null, vCols, dTFactory, false); + } + + public ExprNodeConverter(String tabAlias, RelDataType inputRowType, + Set vCols, RelDataTypeFactory dTFactory, boolean foldExpr) { + this(tabAlias, null, inputRowType, null, vCols, dTFactory, foldExpr); } public ExprNodeConverter(String tabAlias, String columnAlias, RelDataType inputRowType, RelDataType outputRowType, Set inputVCols, RelDataTypeFactory dTFactory) { + this(tabAlias, columnAlias, inputRowType, outputRowType, inputVCols, dTFactory, false); + } + + public ExprNodeConverter(String tabAlias, String columnAlias, RelDataType inputRowType, + RelDataType outputRowType, Set inputVCols, RelDataTypeFactory dTFactory, + boolean foldExpr) { super(true); this.tabAlias = tabAlias; this.inputRowType = inputRowType; this.inputVCols = ImmutableSet.copyOf(inputVCols); this.dTFactory = dTFactory; + this.foldExpr = foldExpr; } public List getWindowFunctionSpec() { @@ -117,7 +132,7 @@ public ExprNodeDesc visitInputRef(RexInputRef inputRef) { */ @Override public ExprNodeDesc visitCall(RexCall call) { - ExprNodeGenericFuncDesc gfDesc = null; + ExprNodeDesc gfDesc = null; if (!deep) { return null; @@ -149,6 +164,7 @@ public ExprNodeDesc visitCall(RexCall call) { throw new RuntimeException(e); } } + return gfDesc; } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java index 7fbf8cd..3ecbbb1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java @@ -159,7 +159,8 @@ private static GBInfo getGBInfo(HiveAggregate aggRel, OpAttr inputOpAf, HiveConf // 1. Collect GB Keys RelNode aggInputRel = aggRel.getInput(); ExprNodeConverter exprConv = new ExprNodeConverter(inputOpAf.tabAlias, - aggInputRel.getRowType(), new HashSet(), aggRel.getCluster().getTypeFactory()); + aggInputRel.getRowType(), new HashSet(), aggRel.getCluster().getTypeFactory(), + true); ExprNodeDesc tmpExprNodeDesc; for (int i : aggRel.getGroupSet()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java index 1307808..aef5baa 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java @@ -278,7 +278,8 @@ OpAttr visit(HiveProject projectRel) throws SemanticException { for (int pos = 0; pos < projectRel.getChildExps().size(); pos++) { ExprNodeConverter converter = new ExprNodeConverter(inputOpAf.tabAlias, projectRel .getRowType().getFieldNames().get(pos), projectRel.getInput().getRowType(), - projectRel.getRowType(), inputOpAf.vcolsInCalcite, projectRel.getCluster().getTypeFactory()); + projectRel.getRowType(), inputOpAf.vcolsInCalcite, projectRel.getCluster().getTypeFactory(), + true); ExprNodeDesc exprCol = projectRel.getChildExps().get(pos).accept(converter); colExprMap.put(exprNames.get(pos), exprCol); exprCols.add(exprCol); @@ -520,7 +521,7 @@ OpAttr visit(HiveFilter filterRel) throws SemanticException { ExprNodeDesc filCondExpr = filterRel.getCondition().accept( new ExprNodeConverter(inputOpAf.tabAlias, filterRel.getInput().getRowType(), inputOpAf.vcolsInCalcite, - filterRel.getCluster().getTypeFactory())); + filterRel.getCluster().getTypeFactory(), true)); FilterDesc filDesc = new FilterDesc(filCondExpr, false); ArrayList cinfoLst = createColInfos(inputOpAf.inputs.get(0)); FilterOperator filOp = (FilterOperator) OperatorFactory.getAndMakeChild(filDesc, @@ -1164,7 +1165,7 @@ private static ExprNodeDesc convertToExprNode(RexNode rn, RelNode inputRel, Stri private static ExprNodeDesc convertToExprNode(RexNode rn, RelNode inputRel, String tabAlias, Set vcolsInCalcite) { return rn.accept(new ExprNodeConverter(tabAlias, inputRel.getRowType(), vcolsInCalcite, - inputRel.getCluster().getTypeFactory())); + inputRel.getCluster().getTypeFactory(), true)); } private static ArrayList createColInfos(Operator input) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java index 368264c..b5f4ca3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java @@ -115,7 +115,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, "In return path join annotate rule, we find " + aliases == null ? null : aliases .size() + " aliases for " + joinOp.toString()); } - final String joinOpAlias = aliases.iterator().next();; + final String joinOpAlias = aliases.iterator().next(); aliasToOpInfo.put(joinOpAlias, joinOp); // 3. Populate other data structures diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java index ee4f4ea..4ed4c6e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java @@ -74,7 +74,9 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToChar; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDate; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDecimal; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToVarchar; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; @@ -157,7 +159,7 @@ private RexNode convert(final ExprNodeFieldDesc fieldDesc) throws SemanticExcept } } - private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticException { + private RexNode convert(ExprNodeGenericFuncDesc func) throws SemanticException { ExprNodeDesc tmpExprNode; RexNode tmpRN; @@ -174,6 +176,8 @@ private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticExcep ((PrimitiveTypeInfo) func.getTypeInfo()).getPrimitiveCategory()))); boolean isCompare = !isNumeric && tgtUdf instanceof GenericUDFBaseCompare; boolean isWhenCase = tgtUdf instanceof GenericUDFWhen || tgtUdf instanceof GenericUDFCase; + boolean isTransformableTimeStamp = func.getGenericUDF() instanceof GenericUDFUnixTimeStamp && + func.getChildren().size() != 0; if (isNumeric) { tgtDT = func.getTypeInfo(); @@ -189,6 +193,9 @@ private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticExcep if (checkForStatefulFunctions(func.getChildren())) { throw new SemanticException("Stateful expressions cannot be used inside of CASE"); } + } else if (isTransformableTimeStamp) { + // unix_timestamp(args) -> to_unix_timestamp(args) + func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFToUnixTimeStamp(), func.getChildren()); } for (ExprNodeDesc childExpr : func.getChildren()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java index 0b76bff..8b08ae7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java @@ -37,8 +37,6 @@ import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.util.Util; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.FunctionInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -65,6 +63,8 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -204,6 +204,7 @@ public static ASTNode buildAST(SqlOperator op, List children) { case BETWEEN: case ROW: case IS_NOT_NULL: + case IS_NULL: case CASE: node = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_FUNCTION, "TOK_FUNCTION"); node.addChild((ASTNode) ParseDriver.adaptor.create(hToken.type, hToken.text)); @@ -322,13 +323,15 @@ private static String getName(GenericUDF hiveUDF) { registerFunction(">", SqlStdOperatorTable.GREATER_THAN, hToken(HiveParser.GREATERTHAN, ">")); registerFunction(">=", SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, hToken(HiveParser.GREATERTHANOREQUALTO, ">=")); - registerFunction("!", SqlStdOperatorTable.NOT, hToken(HiveParser.KW_NOT, "not")); + registerFunction("not", SqlStdOperatorTable.NOT, hToken(HiveParser.KW_NOT, "not")); + registerDuplicateFunction("!", SqlStdOperatorTable.NOT, hToken(HiveParser.KW_NOT, "not")); registerFunction("<>", SqlStdOperatorTable.NOT_EQUALS, hToken(HiveParser.NOTEQUAL, "<>")); registerDuplicateFunction("!=", SqlStdOperatorTable.NOT_EQUALS, hToken(HiveParser.NOTEQUAL, "<>")); registerFunction("in", HiveIn.INSTANCE, hToken(HiveParser.Identifier, "in")); registerFunction("between", HiveBetween.INSTANCE, hToken(HiveParser.Identifier, "between")); registerFunction("struct", SqlStdOperatorTable.ROW, hToken(HiveParser.Identifier, "struct")); registerFunction("isnotnull", SqlStdOperatorTable.IS_NOT_NULL, hToken(HiveParser.TOK_ISNOTNULL, "TOK_ISNOTNULL")); + registerFunction("isnull", SqlStdOperatorTable.IS_NULL, hToken(HiveParser.TOK_ISNULL, "TOK_ISNULL")); registerFunction("when", SqlStdOperatorTable.CASE, hToken(HiveParser.Identifier, "when")); registerDuplicateFunction("case", SqlStdOperatorTable.CASE, hToken(HiveParser.Identifier, "when")); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java index 2825f77..ba41518 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.translator; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; @@ -28,9 +29,11 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.SqlCollation; import org.apache.calcite.sql.SqlIntervalQualifier; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.ConversionUtil; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -57,6 +60,7 @@ import com.google.common.collect.Lists; public class TypeConverter { + private static final Map calciteToHiveTypeNameMap; // TODO: Handling of char[], varchar[], string... @@ -162,7 +166,9 @@ public static RelDataType convert(PrimitiveTypeInfo type, RelDataTypeFactory dtF convertedType = dtFactory.createSqlType(SqlTypeName.DOUBLE); break; case STRING: - convertedType = dtFactory.createSqlType(SqlTypeName.VARCHAR, Integer.MAX_VALUE); + convertedType = dtFactory.createTypeWithCharsetAndCollation( + dtFactory.createSqlType(SqlTypeName.VARCHAR, Integer.MAX_VALUE), + Charset.forName(ConversionUtil.NATIVE_UTF16_CHARSET_NAME), SqlCollation.IMPLICIT); break; case DATE: convertedType = dtFactory.createSqlType(SqlTypeName.DATE); @@ -187,12 +193,14 @@ public static RelDataType convert(PrimitiveTypeInfo type, RelDataTypeFactory dtF .createSqlType(SqlTypeName.DECIMAL, dtInf.precision(), dtInf.scale()); break; case VARCHAR: - convertedType = dtFactory.createSqlType(SqlTypeName.VARCHAR, - ((BaseCharTypeInfo) type).getLength()); + convertedType = dtFactory.createTypeWithCharsetAndCollation( + dtFactory.createSqlType(SqlTypeName.VARCHAR, ((BaseCharTypeInfo) type).getLength()), + Charset.forName(ConversionUtil.NATIVE_UTF16_CHARSET_NAME), SqlCollation.IMPLICIT); break; case CHAR: - convertedType = dtFactory.createSqlType(SqlTypeName.CHAR, - ((BaseCharTypeInfo) type).getLength()); + convertedType = dtFactory.createTypeWithCharsetAndCollation( + dtFactory.createSqlType(SqlTypeName.CHAR, ((BaseCharTypeInfo) type).getLength()), + Charset.forName(ConversionUtil.NATIVE_UTF16_CHARSET_NAME), SqlCollation.IMPLICIT); break; case UNKNOWN: convertedType = dtFactory.createSqlType(SqlTypeName.OTHER); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java index 9911179..f9388e2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; +import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; import org.apache.hadoop.hive.ql.optimizer.ppr.PartExprEvalUtils; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -474,7 +475,20 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, new ExprNodeConstantDesc(fd.getTypeInfo(), result)); } - return new NodeInfoWrapper(WalkState.CONSTANT, null, getOutExpr(fd, nodeOutputs)); + // Try to fold, otherwise return the expression itself + final ExprNodeGenericFuncDesc desc = getOutExpr(fd, nodeOutputs); + final ExprNodeDesc foldedDesc = ConstantPropagateProcFactory.foldExpr(desc); + if (foldedDesc != null && foldedDesc instanceof ExprNodeConstantDesc) { + ExprNodeConstantDesc constant = (ExprNodeConstantDesc) foldedDesc; + if (Boolean.TRUE.equals(constant.getValue())) { + return new NodeInfoWrapper(WalkState.TRUE, null, constant); + } else if (Boolean.FALSE.equals(constant.getValue())) { + return new NodeInfoWrapper(WalkState.FALSE, null, constant); + } else { + return new NodeInfoWrapper(WalkState.CONSTANT, null, constant); + } + } + return new NodeInfoWrapper(WalkState.CONSTANT, null, desc); } } }; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index de6a053..49e65e7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -140,6 +140,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregateJoinTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregateProjectMergeRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregatePullUpConstantsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveExpandDistinctAggregatesRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterAggregateTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveFilterJoinRule; @@ -153,21 +154,22 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinProjectTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinPushTransitivePredicatesRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinToMultiJoinRule; -import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortLimitPullUpConstantsRule; -import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveUnionPullUpConstantsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePartitionPruneRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePointLookupOptimizerRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePreFilteringRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectFilterPullUpConstantsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectMergeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectSortTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveReduceExpressionsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRelFieldTrimmer; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRulesRegistry; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortJoinReduceRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortLimitPullUpConstantsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortMergeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortProjectTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortRemoveRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortUnionReduceRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveUnionPullUpConstantsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveWindowingFixRule; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverter; @@ -1154,6 +1156,7 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv rules.add(HiveFilterJoinRule.FILTER_ON_JOIN); rules.add(new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class)); rules.add(new FilterMergeRule(HiveRelFactories.HIVE_FILTER_FACTORY)); + rules.add(HiveProjectFilterPullUpConstantsRule.INSTANCE); rules.add(HiveReduceExpressionsRule.PROJECT_INSTANCE); rules.add(HiveReduceExpressionsRule.FILTER_INSTANCE); rules.add(HiveReduceExpressionsRule.JOIN_INSTANCE); @@ -1168,6 +1171,7 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv rules.add(HiveSortMergeRule.INSTANCE); rules.add(HiveSortLimitPullUpConstantsRule.INSTANCE); rules.add(HiveUnionPullUpConstantsRule.INSTANCE); + rules.add(HiveAggregatePullUpConstantsRule.INSTANCE); perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, executorProvider, HepMatchOrder.BOTTOM_UP, rules.toArray(new RelOptRule[rules.size()])); @@ -1217,10 +1221,10 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Projection Pruning"); - // 8. Merge Project-Project if possible + // 8. Merge, remove and reduce Project if possible perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - basePlan = hepPlan(basePlan, false, mdProvider, null, new ProjectMergeRule(true, - HiveRelFactories.HIVE_PROJECT_FACTORY)); + basePlan = hepPlan(basePlan, false, mdProvider, null, + HiveProjectMergeRule.INSTANCE, ProjectRemoveRule.INSTANCE); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Merge Project-Project"); @@ -1229,9 +1233,11 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv // storage (incase there are filters on non partition cols). This only // matches FIL-PROJ-TS perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - basePlan = hepPlan(basePlan, true, mdProvider, null, new HiveFilterProjectTSTransposeRule( - Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, - HiveRelFactories.HIVE_PROJECT_FACTORY, HiveTableScan.class)); + basePlan = hepPlan(basePlan, true, mdProvider, null, + new HiveFilterProjectTSTransposeRule( + Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class, + HiveRelFactories.HIVE_PROJECT_FACTORY, HiveTableScan.class), + HiveProjectFilterPullUpConstantsRule.INSTANCE); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Prejoin ordering transformation, Rerun PPD"); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 7162c08..6d20c5a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -35,8 +35,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Queue; import java.util.Map.Entry; +import java.util.Queue; import java.util.Set; import java.util.TreeSet; import java.util.UUID; @@ -71,7 +71,6 @@ import org.apache.hadoop.hive.metastore.api.SQLForeignKey; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.QueryState; @@ -194,7 +193,6 @@ import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; -import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.ResourceType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; @@ -3142,9 +3140,23 @@ private Operator genFilterPlan(QB qb, ASTNode condn, Operator input, boolean use OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); + + ExprNodeDesc filterCond = genExprNodeDesc(condn, inputRR, useCaching, isCBOExecuted()); + if (filterCond instanceof ExprNodeConstantDesc) { + ExprNodeConstantDesc c = (ExprNodeConstantDesc) filterCond; + if (Boolean.TRUE.equals(c.getValue())) { + // If filter condition is TRUE, we ignore it + return input; + } + if (ExprNodeDescUtils.isNullConstant(c)) { + // If filter condition is NULL, transform to FALSE + filterCond = new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, false); + } + } + Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( - new FilterDesc(genExprNodeDesc(condn, inputRR, useCaching, isCBOExecuted()), false), - new RowSchema(inputRR.getColumnInfos()), input), inputRR); + new FilterDesc(filterCond, false), new RowSchema( + inputRR.getColumnInfos()), input), inputRR); if (LOG.isDebugEnabled()) { LOG.debug("Created Filter Plan for " + qb.getId() + " row schema: " diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 2eaed56..239cc61 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -52,10 +52,12 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.udf.SettableUDF; @@ -1070,6 +1072,17 @@ protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, desc = ExprNodeGenericFuncDesc.newInstance(genericUDF, funcText, children); } + + // If the function is deterministic and the children are constants, + // we try to fold the expression to remove e.g. cast on constant + if (ctx.isFoldExpr() && desc instanceof ExprNodeGenericFuncDesc && + FunctionRegistry.isDeterministic(genericUDF) && + ExprNodeDescUtils.isAllConstants(children)) { + ExprNodeDesc constantExpr = ConstantPropagateProcFactory.foldExpr((ExprNodeGenericFuncDesc)desc); + if (constantExpr != null) { + desc = constantExpr; + } + } } // UDFOPPositive is a no-op. // However, we still create it, and then remove it here, to make sure we diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java index c6f8907..2b7b0c3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; @@ -210,15 +211,25 @@ public static boolean isDeterministic(ExprNodeDesc desc) { */ public static ArrayList backtrack(List sources, Operator current, Operator terminal) throws SemanticException { + return backtrack(sources, current, terminal, false); + } + + public static ArrayList backtrack(List sources, + Operator current, Operator terminal, boolean foldExpr) throws SemanticException { ArrayList result = new ArrayList(); for (ExprNodeDesc expr : sources) { - result.add(backtrack(expr, current, terminal)); + result.add(backtrack(expr, current, terminal, foldExpr)); } return result; } public static ExprNodeDesc backtrack(ExprNodeDesc source, Operator current, Operator terminal) throws SemanticException { + return backtrack(source, current, terminal, false); + } + + public static ExprNodeDesc backtrack(ExprNodeDesc source, Operator current, + Operator terminal, boolean foldExpr) throws SemanticException { Operator parent = getSingleParent(current, terminal); if (parent == null) { return source; @@ -226,7 +237,7 @@ public static ExprNodeDesc backtrack(ExprNodeDesc source, Operator current, if (source instanceof ExprNodeGenericFuncDesc) { // all children expression should be resolved ExprNodeGenericFuncDesc function = (ExprNodeGenericFuncDesc) source.clone(); - List children = backtrack(function.getChildren(), current, terminal); + List children = backtrack(function.getChildren(), current, terminal, foldExpr); for (ExprNodeDesc child : children) { if (child == null) { // Could not resolve all of the function children, fail @@ -234,6 +245,13 @@ public static ExprNodeDesc backtrack(ExprNodeDesc source, Operator current, } } function.setChildren(children); + if (foldExpr) { + // fold after replacing, if possible + ExprNodeDesc foldedFunction = ConstantPropagateProcFactory.foldExpr(function); + if (foldedFunction != null) { + return foldedFunction; + } + } return function; } if (source instanceof ExprNodeColumnDesc) { @@ -243,7 +261,7 @@ public static ExprNodeDesc backtrack(ExprNodeDesc source, Operator current, if (source instanceof ExprNodeFieldDesc) { // field expression should be resolved ExprNodeFieldDesc field = (ExprNodeFieldDesc) source.clone(); - ExprNodeDesc fieldDesc = backtrack(field.getDesc(), current, terminal); + ExprNodeDesc fieldDesc = backtrack(field.getDesc(), current, terminal, foldExpr); if (fieldDesc == null) { return null; } @@ -485,6 +503,25 @@ public static void getExprNodeColumnDesc(ExprNodeDesc exprDesc, } } + public static boolean isConstant(ExprNodeDesc value) { + if (value instanceof ExprNodeConstantDesc) { + return true; + } + if (value instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc func = (ExprNodeGenericFuncDesc) value; + if (!FunctionRegistry.isDeterministic(func.getGenericUDF())) { + return false; + } + for (ExprNodeDesc child : func.getChildren()) { + if (!isConstant(child)) { + return false; + } + } + return true; + } + return false; + } + public static boolean isAllConstants(List value) { for (ExprNodeDesc expr : value) { if (!(expr instanceof ExprNodeConstantDesc)) { @@ -641,4 +678,35 @@ public static ExprNodeColumnDesc getColumnExpr(ExprNodeDesc expr) { } return (expr instanceof ExprNodeColumnDesc) ? (ExprNodeColumnDesc)expr : null; } + + // Find the constant origin of a certain column if it is originated from a constant + // Otherwise, it returns the expression that originated the column + public static ExprNodeDesc findConstantExprOrigin(String dpCol, Operator op) { + ExprNodeDesc expr = op.getColumnExprMap().get(dpCol); + ExprNodeDesc foldedExpr; + // If it is a function, we try to fold it + if (expr instanceof ExprNodeGenericFuncDesc) { + foldedExpr = ConstantPropagateProcFactory.foldExpr((ExprNodeGenericFuncDesc)expr); + if (foldedExpr == null) { + foldedExpr = expr; + } + } else { + foldedExpr = expr; + } + // If it is a column reference, we will try to resolve it + if (foldedExpr instanceof ExprNodeColumnDesc) { + Operator originOp = null; + for(Operator parentOp : op.getParentOperators()) { + if (parentOp.getColumnExprMap() != null) { + originOp = parentOp; + break; + } + } + if (originOp != null) { + return findConstantExprOrigin(((ExprNodeColumnDesc)foldedExpr).getColumn(), originOp); + } + } + // Otherwise, we return the expression + return foldedExpr; + } } diff --git ql/src/test/queries/clientpositive/join_view.q ql/src/test/queries/clientpositive/join_view.q index 16b6816..69c96be 100644 --- ql/src/test/queries/clientpositive/join_view.q +++ ql/src/test/queries/clientpositive/join_view.q @@ -3,8 +3,6 @@ drop table invites2; create table invites (foo int, bar string) partitioned by (ds string); create table invites2 (foo int, bar string) partitioned by (ds string); -set hive.mapred.mode=strict; - -- test join views: see HIVE-1989 create view v as select invites.bar, invites2.foo, invites2.ds from invites join invites2 on invites.ds=invites2.ds; @@ -13,4 +11,4 @@ explain select * from v where ds='2011-09-01'; drop view v; drop table invites; -drop table invites2; \ No newline at end of file +drop table invites2; diff --git ql/src/test/results/clientpositive/annotate_stats_filter.q.out ql/src/test/results/clientpositive/annotate_stats_filter.q.out index ba0419e..5db266e 100644 --- ql/src/test/results/clientpositive/annotate_stats_filter.q.out +++ ql/src/test/results/clientpositive/annotate_stats_filter.q.out @@ -718,15 +718,15 @@ STAGE PLANS: alias: loc_orc Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((year = 2001) and year is null) (type: boolean) + predicate: false (type: boolean) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: state (type: string), locid (type: int), zip (type: bigint), null (type: int) + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -759,7 +759,7 @@ STAGE PLANS: predicate: ((year = 2001) and (state = 'OH') and (state = 'FL')) (type: boolean) Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'FL' (type: string), locid (type: int), zip (type: bigint), 2001 (type: int) + expressions: state (type: string), locid (type: int), zip (type: bigint), 2001 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git ql/src/test/results/clientpositive/archive_excludeHadoop20.q.out ql/src/test/results/clientpositive/archive_excludeHadoop20.q.out index c2b9872..52d17b4 100644 --- ql/src/test/results/clientpositive/archive_excludeHadoop20.q.out +++ ql/src/test/results/clientpositive/archive_excludeHadoop20.q.out @@ -137,6 +137,7 @@ POSTHOOK: Input: default@tstsrcpart POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 0 3 +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM tstsrcpart a JOIN tstsrc b ON a.key=b.key WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/archive_multi.q.out ql/src/test/results/clientpositive/archive_multi.q.out index 0ad29d1..38f3f1a 100644 --- ql/src/test/results/clientpositive/archive_multi.q.out +++ ql/src/test/results/clientpositive/archive_multi.q.out @@ -141,6 +141,7 @@ POSTHOOK: Input: ac_test@tstsrcpart POSTHOOK: Input: ac_test@tstsrcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 0 3 +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM ac_test.tstsrcpart a JOIN ac_test.tstsrc b ON a.key=b.key WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/authorization_explain.q.java1.7.out ql/src/test/results/clientpositive/authorization_explain.q.java1.7.out index a9ed049..fefb50c 100644 --- ql/src/test/results/clientpositive/authorization_explain.q.java1.7.out +++ ql/src/test/results/clientpositive/authorization_explain.q.java1.7.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- JAVA_VERSION_SPECIFIC_OUTPUT explain authorization select * from src join srcpart @@ -20,7 +20,7 @@ CURRENT_USER: hive_test_user OPERATION: QUERY -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain formatted authorization select * from src join srcpart PREHOOK: type: QUERY POSTHOOK: query: explain formatted authorization select * from src join srcpart diff --git ql/src/test/results/clientpositive/auto_join33.q.out ql/src/test/results/clientpositive/auto_join33.q.out index b0b3019..5a8bf8c 100644 --- ql/src/test/results/clientpositive/auto_join33.q.out +++ ql/src/test/results/clientpositive/auto_join33.q.out @@ -42,8 +42,8 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 (UDFToDouble(_col0) + UDFToDouble(1)) (type: double) - 1 (UDFToDouble(_col0) + UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) + 1.0) (type: double) + 1 (UDFToDouble(_col0) + 2.0) (type: double) Stage: Stage-3 Map Reduce @@ -62,8 +62,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) + UDFToDouble(1)) (type: double) - 1 (UDFToDouble(_col0) + UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) + 1.0) (type: double) + 1 (UDFToDouble(_col0) + 2.0) (type: double) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/auto_join8.q.out ql/src/test/results/clientpositive/auto_join8.q.out index 324f95d..8daa1c5 100644 --- ql/src/test/results/clientpositive/auto_join8.q.out +++ ql/src/test/results/clientpositive/auto_join8.q.out @@ -152,7 +152,7 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.c3 EXPRESSION [] POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/auto_join_filters.q.out ql/src/test/results/clientpositive/auto_join_filters.q.out index 2fdf470..2d4a043 100644 --- ql/src/test/results/clientpositive/auto_join_filters.q.out +++ ql/src/test/results/clientpositive/auto_join_filters.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -300,7 +300,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in2.txt' into table sm POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_input2 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/auto_join_nulls.q.out ql/src/test/results/clientpositive/auto_join_nulls.q.out index 4af5535..44917c5 100644 --- ql/src/test/results/clientpositive/auto_join_nulls.q.out +++ ql/src/test/results/clientpositive/auto_join_nulls.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -24,7 +24,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out index d8eacbe..62c819e 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out @@ -138,7 +138,7 @@ POSTHOOK: query: load data local inpath '../../data/files/smallsrcsortbucket3out POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@bucket_medium@ds=2008-04-08 -Warning: Map Join MAPJOIN[32][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[31][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY POSTHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key @@ -631,7 +631,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[32][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[31][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@bucket_big diff --git ql/src/test/results/clientpositive/bucket_groupby.q.out ql/src/test/results/clientpositive/bucket_groupby.q.out index ae736f9..1afab38 100644 --- ql/src/test/results/clientpositive/bucket_groupby.q.out +++ ql/src/test/results/clientpositive/bucket_groupby.q.out @@ -438,33 +438,29 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - keys: _col0 (type: string), 3 (type: int) + keys: _col0 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), 3 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), 3 (type: int) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), 3 (type: int) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col2 (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -1018,34 +1014,30 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - keys: _col0 (type: string), 3 (type: int) + keys: _col0 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), 3 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), 3 (type: int) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), 3 (type: int) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col2 (type: bigint) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/cast1.q.out ql/src/test/results/clientpositive/cast1.q.out index 48a0c14..d87c04c 100644 --- ql/src/test/results/clientpositive/cast1.q.out +++ ql/src/test/results/clientpositive/cast1.q.out @@ -110,8 +110,8 @@ POSTHOOK: Lineage: dest1.c2 SIMPLE [] POSTHOOK: Lineage: dest1.c3 SIMPLE [] POSTHOOK: Lineage: dest1.c4 SIMPLE [] POSTHOOK: Lineage: dest1.c5 SIMPLE [] -POSTHOOK: Lineage: dest1.c6 EXPRESSION [] -POSTHOOK: Lineage: dest1.c7 EXPRESSION [] +POSTHOOK: Lineage: dest1.c6 SIMPLE [] +POSTHOOK: Lineage: dest1.c7 SIMPLE [] PREHOOK: query: select dest1.* FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 diff --git ql/src/test/results/clientpositive/cbo_const.q.out ql/src/test/results/clientpositive/cbo_const.q.out index c2a5194..ecf0269 100644 --- ql/src/test/results/clientpositive/cbo_const.q.out +++ ql/src/test/results/clientpositive/cbo_const.q.out @@ -19,21 +19,18 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### 2 01:02:03.000000000 2 01:02:03.000000000 2 01:02:03.000000000 2 01:02:03.000000000 true +Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: drop view t1 diff --git ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out index a1be9b9d..e19bb9e 100644 --- ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out +++ ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out @@ -362,11 +362,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 32 Data size: 5600 Basic stats: COMPLETE Column stats: COMPLETE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 32 Data size: 5600 Basic stats: COMPLETE Column stats: COMPLETE @@ -420,11 +419,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 24 Data size: 4200 Basic stats: COMPLETE Column stats: COMPLETE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 24 Data size: 4200 Basic stats: COMPLETE Column stats: COMPLETE @@ -478,11 +476,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 8 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE @@ -536,11 +533,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 16 Data size: 2800 Basic stats: COMPLETE Column stats: COMPLETE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 16 Data size: 2800 Basic stats: COMPLETE Column stats: COMPLETE @@ -594,11 +590,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 24 Data size: 4200 Basic stats: COMPLETE Column stats: COMPLETE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 24 Data size: 4200 Basic stats: COMPLETE Column stats: COMPLETE @@ -652,11 +647,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 32 Data size: 5600 Basic stats: COMPLETE Column stats: COMPLETE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 32 Data size: 5600 Basic stats: COMPLETE Column stats: COMPLETE @@ -767,11 +761,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 16 Data size: 2800 Basic stats: COMPLETE Column stats: COMPLETE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 16 Data size: 2800 Basic stats: COMPLETE Column stats: COMPLETE @@ -880,11 +873,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 32 Data size: 3184 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 32 Data size: 3184 Basic stats: COMPLETE Column stats: NONE @@ -938,11 +930,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 24 Data size: 2388 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 24 Data size: 2388 Basic stats: COMPLETE Column stats: NONE @@ -996,11 +987,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE @@ -1054,11 +1044,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE @@ -1112,11 +1101,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 24 Data size: 2388 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 24 Data size: 2388 Basic stats: COMPLETE Column stats: NONE @@ -1170,11 +1158,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 32 Data size: 3184 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 32 Data size: 3184 Basic stats: COMPLETE Column stats: NONE @@ -1281,11 +1268,10 @@ STAGE PLANS: Group By Operator keys: state (type: string), locid (type: int), '0' (type: string) mode: hash - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 32 Data size: 3184 Basic stats: COMPLETE Column stats: NONE - pruneGroupingSetId: true Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), '0' (type: string) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 32 Data size: 3184 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/cbo_rp_join1.q.out ql/src/test/results/clientpositive/cbo_rp_join1.q.out index f3982b8..01a367e 100644 --- ql/src/test/results/clientpositive/cbo_rp_join1.q.out +++ ql/src/test/results/clientpositive/cbo_rp_join1.q.out @@ -141,7 +141,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col1 = 40) and (_col0 = 40)) (type: boolean) + predicate: ((_col0 = _col1) and (_col1 = 40) and (_col0 = 40)) (type: boolean) Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: diff --git ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out index 2d3f12b..9e0613f 100644 --- ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out +++ ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out @@ -18,7 +18,7 @@ PREHOOK: query: select * from src1 where key > 10 and value > 'val' order by key PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"773d9d0ea92e797eae292ae1eeea11ab","queryText":"select * from src1 where key > 10 and value > 'val' order by key limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2,3],"targets":[0,1],"expression":"((UDFToDouble(src1.key) > UDFToDouble(10)) and (src1.value > 'val'))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"src1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"src1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"773d9d0ea92e797eae292ae1eeea11ab","queryText":"select * from src1 where key > 10 and value > 'val' order by key limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2,3],"targets":[0,1],"expression":"((UDFToDouble(src1.key) > 10.0) and (src1.value > 'val'))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"src1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"src1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} 146 val_146 150 val_150 213 val_213 diff --git ql/src/test/results/clientpositive/colstats_all_nulls.q.out ql/src/test/results/clientpositive/colstats_all_nulls.q.out index d567ec8..f67f81b 100644 --- ql/src/test/results/clientpositive/colstats_all_nulls.q.out +++ ql/src/test/results/clientpositive/colstats_all_nulls.q.out @@ -24,9 +24,9 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src_null POSTHOOK: Output: database:default POSTHOOK: Output: default@all_nulls -POSTHOOK: Lineage: all_nulls.a SIMPLE [(src_null)src_null.FieldSchema(name:a, type:bigint, comment:null), ] -POSTHOOK: Lineage: all_nulls.b EXPRESSION [(src_null)src_null.FieldSchema(name:a, type:bigint, comment:null), ] -POSTHOOK: Lineage: all_nulls.c EXPRESSION [(src_null)src_null.FieldSchema(name:a, type:bigint, comment:null), ] +POSTHOOK: Lineage: all_nulls.a SIMPLE [] +POSTHOOK: Lineage: all_nulls.b SIMPLE [] +POSTHOOK: Lineage: all_nulls.c SIMPLE [] PREHOOK: query: analyze table all_nulls compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@all_nulls diff --git ql/src/test/results/clientpositive/constantPropagateForSubQuery.q.out ql/src/test/results/clientpositive/constantPropagateForSubQuery.q.out index c7a39f5..2aa8d77 100644 --- ql/src/test/results/clientpositive/constantPropagateForSubQuery.q.out +++ ql/src/test/results/clientpositive/constantPropagateForSubQuery.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- SORT_QUERY_RESULTS explain extended @@ -188,7 +188,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from (select a.key as ak, a.value as av, b.key as bk, b.value as bv from src a join src1 b where a.key = '429' ) c PREHOOK: type: QUERY PREHOOK: Input: default@src diff --git ql/src/test/results/clientpositive/constant_prop_3.q.out ql/src/test/results/clientpositive/constant_prop_3.q.out index 3635913..fe382fc 100644 --- ql/src/test/results/clientpositive/constant_prop_3.q.out +++ ql/src/test/results/clientpositive/constant_prop_3.q.out @@ -88,7 +88,7 @@ POSTHOOK: query: analyze table supplier_hive compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@supplier_hive #### A masked pattern was here #### -Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select p_brand, p_type, diff --git ql/src/test/results/clientpositive/constprog3.q.out ql/src/test/results/clientpositive/constprog3.q.out index e01a733..3fd776e 100644 --- ql/src/test/results/clientpositive/constprog3.q.out +++ ql/src/test/results/clientpositive/constprog3.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: create temporary table table3(id int, val int, val1 int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@table3 -Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 inner join table3 @@ -49,15 +49,15 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int) TableScan alias: table3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (id = 1) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Join Operator condition map: diff --git ql/src/test/results/clientpositive/constprog_semijoin.q.out ql/src/test/results/clientpositive/constprog_semijoin.q.out index 35d062d..1940987 100644 --- ql/src/test/results/clientpositive/constprog_semijoin.q.out +++ ql/src/test/results/clientpositive/constprog_semijoin.q.out @@ -421,7 +421,7 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid <> 100) and (dimid = 100) is not null) (type: boolean) + predicate: (((dimid = 100) = true) and (dimid <> 100)) (type: boolean) Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) @@ -437,10 +437,10 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id <> 100) and (id = 100) is not null) (type: boolean) + predicate: (((id = 100) = true) and (id <> 100)) (type: boolean) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), (id = 100) (type: boolean) + expressions: id (type: int), true (type: boolean) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -502,7 +502,7 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((dimid) IN (100, 200) and ((dimid = 100) = true) and (dimid = 100) is not null) (type: boolean) + predicate: ((dimid) IN (100, 200) and ((dimid = 100) = true)) (type: boolean) Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) @@ -518,10 +518,10 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((id) IN (100, 200) and ((id = 100) = true) and (id = 100) is not null) (type: boolean) + predicate: ((id) IN (100, 200) and ((id = 100) = true)) (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), (id = 100) (type: boolean) + expressions: id (type: int), true (type: boolean) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -585,7 +585,7 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid = 200) and (dimid = 100) is not null) (type: boolean) + predicate: (((dimid = 100) = true) and (dimid = 200)) (type: boolean) Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string) @@ -601,26 +601,28 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id = 200) and (id = 100) is not null) (type: boolean) + predicate: (((id = 100) = true) and (id = 200)) (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator + expressions: 200 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: 200 (type: int), false (type: boolean) + keys: _col0 (type: int), _col1 (type: boolean) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 200 (type: int), false (type: boolean) + key expressions: _col0 (type: int), _col1 (type: boolean) sort order: ++ - Map-reduce partition columns: 200 (type: int), false (type: boolean) + Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col3 (type: int), true (type: boolean) + 0 200 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE @@ -664,7 +666,7 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid = 100) and (dimid = 100) is not null) (type: boolean) + predicate: (((dimid = 100) = true) and (dimid = 100)) (type: boolean) Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string) @@ -680,26 +682,28 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id = 100) and (id = 100) is not null) (type: boolean) + predicate: (((id = 100) = true) and (id = 100)) (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator + expressions: 100 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: 100 (type: int), true (type: boolean) + keys: _col0 (type: int), _col1 (type: boolean) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 100 (type: int), true (type: boolean) + key expressions: _col0 (type: int), _col1 (type: boolean) sort order: ++ - Map-reduce partition columns: 100 (type: int), true (type: boolean) + Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col3 (type: int), true (type: boolean) + 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE @@ -745,7 +749,7 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and dimid is not null and (dimid = 100) is not null) (type: boolean) + predicate: (((dimid = 100) = true) and dimid is not null) (type: boolean) Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) @@ -761,10 +765,10 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and id is not null and (id = 100) is not null) (type: boolean) + predicate: (((id = 100) = true) and id is not null) (type: boolean) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), (id = 100) (type: boolean) + expressions: id (type: int), true (type: boolean) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator diff --git ql/src/test/results/clientpositive/cp_sel.q.out ql/src/test/results/clientpositive/cp_sel.q.out index f42ec84..9ee9754 100644 --- ql/src/test/results/clientpositive/cp_sel.q.out +++ ql/src/test/results/clientpositive/cp_sel.q.out @@ -16,18 +16,18 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + expressions: key (type: string), value (type: string), 'hello' (type: string), 'world' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 1 (type: int) sort order: + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), 'hello' (type: string), 'world' (type: string) + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Limit @@ -87,18 +87,18 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + expressions: key (type: string), value (type: string), 'hello' (type: string), 'world' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), 'hello' (type: string), 'world' (type: string) + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/create_genericudf.q.out ql/src/test/results/clientpositive/create_genericudf.q.out index db3a9b5..b7771b2 100644 --- ql/src/test/results/clientpositive/create_genericudf.q.out +++ ql/src/test/results/clientpositive/create_genericudf.q.out @@ -52,9 +52,9 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.c1 SIMPLE [] POSTHOOK: Lineage: dest1.c2 SIMPLE [] -POSTHOOK: Lineage: dest1.c3 EXPRESSION [] -POSTHOOK: Lineage: dest1.c4 EXPRESSION [] -POSTHOOK: Lineage: dest1.c5 EXPRESSION [] +POSTHOOK: Lineage: dest1.c3 SIMPLE [] +POSTHOOK: Lineage: dest1.c4 SIMPLE [] +POSTHOOK: Lineage: dest1.c5 SIMPLE [] POSTHOOK: Lineage: dest1.c6 SIMPLE [] POSTHOOK: Lineage: dest1.c7 SIMPLE [] PREHOOK: query: SELECT dest1.* FROM dest1 LIMIT 1 diff --git ql/src/test/results/clientpositive/create_view.q.out ql/src/test/results/clientpositive/create_view.q.out index e23a993..809f701 100644 --- ql/src/test/results/clientpositive/create_view.q.out +++ ql/src/test/results/clientpositive/create_view.q.out @@ -559,7 +559,7 @@ POSTHOOK: Input: default@table1 POSTHOOK: Input: default@view4 POSTHOOK: Output: database:default POSTHOOK: Output: default@view5 -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM view5 PREHOOK: type: QUERY PREHOOK: Input: default@table1 diff --git ql/src/test/results/clientpositive/cross_join.q.out ql/src/test/results/clientpositive/cross_join.q.out index f01993d..79ef5b3 100644 --- ql/src/test/results/clientpositive/cross_join.q.out +++ ql/src/test/results/clientpositive/cross_join.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- current explain select src.key from src join src src2 PREHOOK: type: QUERY @@ -55,7 +55,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- ansi cross join explain select src.key from src cross join src src2 PREHOOK: type: QUERY @@ -179,7 +179,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain select src.key from src join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src join src src2 @@ -242,7 +242,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain select src.key from src cross join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src cross join src src2 diff --git ql/src/test/results/clientpositive/cross_join_merge.q.out ql/src/test/results/clientpositive/cross_join_merge.q.out index f15dd17..f0b80a7 100644 --- ql/src/test/results/clientpositive/cross_join_merge.q.out +++ ql/src/test/results/clientpositive/cross_join_merge.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select src1.key from src src1 join src src2 join src src3 PREHOOK: type: QUERY @@ -233,7 +233,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[15][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select src1.key from src src1 join src src2 on 5 = src2.key join src src3 on src1.key=src3.key PREHOOK: type: QUERY @@ -337,7 +337,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select src1.key from src src1 left outer join src src2 join src src3 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/cross_product_check_1.q.out ql/src/test/results/clientpositive/cross_product_check_1.q.out index 4feb798..907319d 100644 --- ql/src/test/results/clientpositive/cross_product_check_1.q.out +++ ql/src/test/results/clientpositive/cross_product_check_1.q.out @@ -32,7 +32,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -90,7 +90,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -194,7 +194,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -330,8 +330,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -Warning: Shuffle Join JOIN[9][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -450,7 +450,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 diff --git ql/src/test/results/clientpositive/cross_product_check_2.q.out ql/src/test/results/clientpositive/cross_product_check_2.q.out index f34f2b5..bb36f84 100644 --- ql/src/test/results/clientpositive/cross_product_check_2.q.out +++ ql/src/test/results/clientpositive/cross_product_check_2.q.out @@ -32,7 +32,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -97,7 +97,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Stage-5:MAPRED' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Stage-5:MAPRED' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -191,7 +191,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Stage-5:MAPRED' is a cross product +Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Stage-5:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -327,8 +327,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[25][bigTable=?] in task 'Stage-5:MAPRED' is a cross product -Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[23][bigTable=?] in task 'Stage-5:MAPRED' is a cross product +Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -450,9 +450,9 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[35][bigTable=?] in task 'Stage-7:MAPRED' is a cross product -Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Stage-6:MAPRED' is a cross product -Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 diff --git ql/src/test/results/clientpositive/cte_5.q.out ql/src/test/results/clientpositive/cte_5.q.out index 96a6543..e9d700d 100644 --- ql/src/test/results/clientpositive/cte_5.q.out +++ ql/src/test/results/clientpositive/cte_5.q.out @@ -118,7 +118,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble('5') (type: double) + 1 5.0 (type: double) outputColumnNames: _col0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/cte_mat_1.q.out ql/src/test/results/clientpositive/cte_mat_1.q.out index b7d5f7d..bb007c1 100644 --- ql/src/test/results/clientpositive/cte_mat_1.q.out +++ ql/src/test/results/clientpositive/cte_mat_1.q.out @@ -1,3 +1,4 @@ +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -27,9 +28,7 @@ STAGE PLANS: Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '5' (type: string) - sort order: + - Map-reduce partition columns: '5' (type: string) + sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: src @@ -40,17 +39,15 @@ STAGE PLANS: Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '5' (type: string) - sort order: + - Map-reduce partition columns: '5' (type: string) + sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 '5' (type: string) - 1 '5' (type: string) + 0 + 1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: '5' (type: string) diff --git ql/src/test/results/clientpositive/cte_mat_2.q.out ql/src/test/results/clientpositive/cte_mat_2.q.out index b7d5f7d..bb007c1 100644 --- ql/src/test/results/clientpositive/cte_mat_2.q.out +++ ql/src/test/results/clientpositive/cte_mat_2.q.out @@ -1,3 +1,4 @@ +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -27,9 +28,7 @@ STAGE PLANS: Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '5' (type: string) - sort order: + - Map-reduce partition columns: '5' (type: string) + sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE TableScan alias: src @@ -40,17 +39,15 @@ STAGE PLANS: Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '5' (type: string) - sort order: + - Map-reduce partition columns: '5' (type: string) + sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 '5' (type: string) - 1 '5' (type: string) + 0 + 1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: '5' (type: string) diff --git ql/src/test/results/clientpositive/decimal_stats.q.out ql/src/test/results/clientpositive/decimal_stats.q.out index 0ea9ed6..72806f0 100644 --- ql/src/test/results/clientpositive/decimal_stats.q.out +++ ql/src/test/results/clientpositive/decimal_stats.q.out @@ -29,7 +29,7 @@ POSTHOOK: query: insert overwrite table decimal_1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@decimal_1 -POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +POSTHOOK: Lineage: decimal_1.t SIMPLE [] POSTHOOK: Lineage: decimal_1.u EXPRESSION [] POSTHOOK: Lineage: decimal_1.v EXPRESSION [] PREHOOK: query: analyze table decimal_1 compute statistics for columns diff --git ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out index 9a09c4c..86346b3 100644 --- ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out +++ ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out @@ -1046,14 +1046,14 @@ STAGE PLANS: outputColumnNames: _col2, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col6 (type: string), _col5 (type: int), _col4 (type: int), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col4 + expressions: _col4 (type: int), _col5 (type: int), _col6 (type: string), _col2 (type: int) + outputColumnNames: _col4, _col5, _col6, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: avg(_col4), stddev_samp(_col4) - keys: _col0 (type: string), _col1 (type: int), _col2 (type: int), 3 (type: int) + aggregations: stddev_samp(_col2), avg(_col2) + keys: _col4 (type: int), _col5 (type: int), _col6 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: true @@ -1067,28 +1067,28 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), 3 (type: int) - sort order: ++++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: int), 3 (type: int) + key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col4 (type: struct), _col5 (type: struct) + value expressions: _col3 (type: struct), _col4 (type: struct) Reduce Operator Tree: Group By Operator - aggregations: avg(VALUE._col0), stddev_samp(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: int), 3 (type: int) + aggregations: stddev_samp(VALUE._col0), avg(VALUE._col1) + keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), _col5 (type: double) - outputColumnNames: _col1, _col2, _col4, _col5 + expressions: _col1 (type: int), _col0 (type: int), _col3 (type: double), _col4 (type: double) + outputColumnNames: _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col5 / _col4) > 1.0)) END (type: boolean) + predicate: CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), CASE WHEN ((_col4 = 0.0)) THEN (null) ELSE ((_col5 / _col4)) END (type: double) - outputColumnNames: _col1, _col2, _col4, _col5 + expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), CASE WHEN ((_col4 = 0.0)) THEN (null) ELSE ((_col3 / _col4)) END (type: double) + outputColumnNames: _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: true @@ -1106,14 +1106,14 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col4 (type: double), _col5 (type: double) + value expressions: _col3 (type: double), _col4 (type: double) TableScan Reduce Output Operator key expressions: _col2 (type: int), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col4 (type: double), _col5 (type: double) + value expressions: _col3 (type: double), _col4 (type: double) Reduce Operator Tree: Join Operator condition map: @@ -1121,10 +1121,10 @@ STAGE PLANS: keys: 0 _col2 (type: int), _col1 (type: int) 1 _col2 (type: int), _col1 (type: int) - outputColumnNames: _col1, _col2, _col4, _col5, _col7, _col8, _col10, _col11 + outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col9 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), _col5 (type: double), _col7 (type: int), _col8 (type: int), _col10 (type: double), _col11 (type: double) + expressions: _col1 (type: int), _col2 (type: int), _col3 (type: double), _col4 (type: double), _col6 (type: int), _col7 (type: int), _col8 (type: double), _col9 (type: double) outputColumnNames: _col0, _col1, _col3, _col4, _col5, _col6, _col8, _col9 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator @@ -1145,7 +1145,7 @@ STAGE PLANS: value expressions: _col5 (type: int), _col6 (type: int) Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), 3 (type: int), KEY.reducesinkkey3 (type: double), KEY.reducesinkkey4 (type: double), VALUE._col0 (type: int), VALUE._col1 (type: int), 4 (type: int), KEY.reducesinkkey6 (type: double), KEY.reducesinkkey7 (type: double) + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: double), KEY.reducesinkkey4 (type: double), VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey5 (type: int), KEY.reducesinkkey6 (type: double), KEY.reducesinkkey7 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator @@ -1283,14 +1283,14 @@ STAGE PLANS: outputColumnNames: _col2, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col6 (type: string), _col5 (type: int), _col4 (type: int), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col4 + expressions: _col4 (type: int), _col5 (type: int), _col6 (type: string), _col2 (type: int) + outputColumnNames: _col4, _col5, _col6, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: avg(_col4), stddev_samp(_col4) - keys: _col0 (type: string), _col1 (type: int), _col2 (type: int), 4 (type: int) + aggregations: stddev_samp(_col2), avg(_col2) + keys: _col4 (type: int), _col5 (type: int), _col6 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: true @@ -1304,28 +1304,28 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), 4 (type: int) - sort order: ++++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: int), 4 (type: int) + key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col4 (type: struct), _col5 (type: struct) + value expressions: _col3 (type: struct), _col4 (type: struct) Reduce Operator Tree: Group By Operator - aggregations: avg(VALUE._col0), stddev_samp(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: int), 4 (type: int) + aggregations: stddev_samp(VALUE._col0), avg(VALUE._col1) + keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), _col5 (type: double) - outputColumnNames: _col1, _col2, _col4, _col5 + expressions: _col1 (type: int), _col0 (type: int), _col3 (type: double), _col4 (type: double) + outputColumnNames: _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col5 / _col4) > 1.0)) END (type: boolean) + predicate: CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), CASE WHEN ((_col4 = 0.0)) THEN (null) ELSE ((_col5 / _col4)) END (type: double) - outputColumnNames: _col1, _col2, _col4, _col5 + expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), CASE WHEN ((_col4 = 0.0)) THEN (null) ELSE ((_col3 / _col4)) END (type: double) + outputColumnNames: _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: true diff --git ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out index 391acff..e0d022f 100644 --- ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out +++ ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out @@ -2267,28 +2267,28 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over1k - Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (s = 'foo') (type: boolean) - Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), t (type: tinyint), i (type: int) - outputColumnNames: _col0, _col1, _col2, _col4, _col5 - Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'foo' (type: string), _col4 (type: tinyint), _col5 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: 'foo' (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), 'foo' (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2332,18 +2332,18 @@ STAGE PLANS: predicate: (t = 27) (type: boolean) Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), i (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col5 + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27 (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string), 27 (type: tinyint), _col5 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: _col3 (type: string), 27 (type: tinyint), _col5 (type: int) + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), 27 (type: tinyint), KEY._col5 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2392,18 +2392,18 @@ STAGE PLANS: predicate: (i = 100) (type: boolean) Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), t (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string), _col4 (type: tinyint), 100 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), 100 (type: int) + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), 100 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2452,18 +2452,18 @@ STAGE PLANS: predicate: ((i = 100) and (t = 27)) (type: boolean) Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27 (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string), 27 (type: tinyint), 100 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: _col3 (type: string), 27 (type: tinyint), 100 (type: int) + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), 27 (type: tinyint), 100 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2507,28 +2507,28 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over1k - Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((i = 100) and (s = 'foo')) (type: boolean) - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), t (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col4 - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'foo' (type: string), _col4 (type: tinyint), 100 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: 'foo' (type: string), _col4 (type: tinyint), 100 (type: int) - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), 'foo' (type: string), KEY._col4 (type: tinyint), 100 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2567,28 +2567,28 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over1k - Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((t = 27) and (s = 'foo')) (type: boolean) - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), i (type: int) - outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27 (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'foo' (type: string), 27 (type: tinyint), _col5 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: 'foo' (type: string), 27 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), 'foo' (type: string), 27 (type: tinyint), KEY._col5 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2632,17 +2632,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over1k - Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((i = 100) and (t = 27) and (s = 'foo')) (type: boolean) - Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 107 Data size: 13282 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27 (type: tinyint), 100 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 107 Data size: 13282 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 107 Data size: 13282 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out index 3b24a2e..387dfee 100644 --- ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out +++ ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out @@ -1538,29 +1538,29 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) - outputColumnNames: _col1, _col2 + outputColumnNames: key, value Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(_col2) - keys: 'day' (type: string), _col1 (type: string) + aggregations: count(value) + keys: key (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'day' (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: 'day' (type: string), _col1 (type: string) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: 'day' (type: string), KEY._col1 (type: string) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int), 'day' (type: string) + expressions: UDFToInteger(_col0) (type: int), UDFToInteger(_col1) (type: int), 'day' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1654,29 +1654,29 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) - outputColumnNames: _col1, _col2 + outputColumnNames: key, value Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(_col2) - keys: 'day' (type: string), _col1 (type: string) + aggregations: count(value) + keys: key (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'day' (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: 'day' (type: string), _col1 (type: string) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: 'day' (type: string), KEY._col1 (type: string) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int), 'day' (type: string) + expressions: UDFToInteger(_col0) (type: int), UDFToInteger(_col1) (type: int), 'day' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/explain_logical.q.out ql/src/test/results/clientpositive/explain_logical.q.out index 5b8a422..3b6f32c 100644 --- ql/src/test/results/clientpositive/explain_logical.q.out +++ ql/src/test/results/clientpositive/explain_logical.q.out @@ -356,35 +356,34 @@ PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN LOGICAL SELECT * FROM V4 POSTHOOK: type: QUERY LOGICAL PLAN: -$hdt$_0:srcpart +$hdt$_0:src TableScan (TS_0) - alias: srcpart - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator (FIL_15) predicate: key is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator (SEL_2) - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator (RS_9) key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Join Operator (JOIN_12) condition map: Inner Join 0 to 1 - Inner Join 0 to 2 + Inner Join 1 to 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) 2 _col0 (type: string) - outputColumnNames: _col1, _col2, _col4 + outputColumnNames: _col0, _col2, _col4 Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE Select Operator (SEL_13) - expressions: _col2 (type: string), _col1 (type: string), _col4 (type: string) + expressions: _col0 (type: string), _col2 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE File Output Operator (FS_14) @@ -394,31 +393,32 @@ $hdt$_0:srcpart input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -$hdt$_1:src +$hdt$_1:srcpart TableScan (TS_3) - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator (FIL_16) predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator (SEL_5) - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator (RS_10) key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Join Operator (JOIN_12) condition map: Inner Join 0 to 1 - Inner Join 0 to 2 + Inner Join 1 to 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) 2 _col0 (type: string) - outputColumnNames: _col1, _col2, _col4 + outputColumnNames: _col0, _col2, _col4 Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE $hdt$_2:src TableScan (TS_6) @@ -440,12 +440,12 @@ $hdt$_2:src Join Operator (JOIN_12) condition map: Inner Join 0 to 1 - Inner Join 0 to 2 + Inner Join 1 to 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) 2 _col0 (type: string) - outputColumnNames: _col1, _col2, _col4 + outputColumnNames: _col0, _col2, _col4 Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE PREHOOK: query: -- The table should show up in the explain logical even if none diff --git ql/src/test/results/clientpositive/filter_cond_pushdown.q.out ql/src/test/results/clientpositive/filter_cond_pushdown.q.out index 132b590..46b701f 100644 --- ql/src/test/results/clientpositive/filter_cond_pushdown.q.out +++ ql/src/test/results/clientpositive/filter_cond_pushdown.q.out @@ -317,7 +317,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double), _col0 (type: string) - 1 UDFToDouble(1) (type: double), _col0 (type: string) + 1 1.0 (type: double), _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col5 Statistics: Num rows: 22 Data size: 288 Basic stats: COMPLETE Column stats: NONE Filter Operator diff --git ql/src/test/results/clientpositive/fold_case.q.out ql/src/test/results/clientpositive/fold_case.q.out index 53139da..165e03a 100644 --- ql/src/test/results/clientpositive/fold_case.q.out +++ ql/src/test/results/clientpositive/fold_case.q.out @@ -267,18 +267,20 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: + Select Operator + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(1) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -371,7 +373,7 @@ STAGE PLANS: predicate: (key <> '238') (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: CASE WHEN ((key = '238')) THEN (null) ELSE (false) END (type: boolean) + expressions: false (type: boolean) outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -405,20 +407,22 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: null (type: void) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: null (type: void) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -444,10 +448,10 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not (key = '238')) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (key <> '238') (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash @@ -497,7 +501,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: null (type: void) + expressions: null (type: string) outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE ListSink diff --git ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out index 106ad6b..13f6ab4 100644 --- ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out +++ ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out @@ -173,10 +173,10 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: CASE WHEN ((key <> '238')) THEN ((key = '238')) ELSE ((key = '238')) END (type: boolean) + predicate: (key = '238') (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) + expressions: '238' (type: string) outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/fold_when.q.out ql/src/test/results/clientpositive/fold_when.q.out index 5b68408..4f3eb14 100644 --- ql/src/test/results/clientpositive/fold_when.q.out +++ ql/src/test/results/clientpositive/fold_when.q.out @@ -156,7 +156,7 @@ STAGE PLANS: predicate: (key = '238') (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) + expressions: '238' (type: string) outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -194,7 +194,7 @@ STAGE PLANS: predicate: (key = '238') (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) + expressions: '238' (type: string) outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/folder_predicate.q.out ql/src/test/results/clientpositive/folder_predicate.q.out index 7fcc172..48a4889 100644 --- ql/src/test/results/clientpositive/folder_predicate.q.out +++ ql/src/test/results/clientpositive/folder_predicate.q.out @@ -37,15 +37,15 @@ STAGE PLANS: alias: predicate_fold_tb Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is null or (value <> 3)) (type: boolean) - Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + predicate: (not (value is not null and (value = 3))) (type: boolean) + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: int) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -88,15 +88,15 @@ STAGE PLANS: alias: predicate_fold_tb Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is null or (value < 3)) (type: boolean) - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + predicate: (not (value is not null and (value >= 3))) (type: boolean) + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: int) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -137,15 +137,15 @@ STAGE PLANS: alias: predicate_fold_tb Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is null or (value > 3)) (type: boolean) - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + predicate: (not (value is not null and (value <= 3))) (type: boolean) + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: int) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -186,15 +186,15 @@ STAGE PLANS: alias: predicate_fold_tb Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is null or (value <= 3)) (type: boolean) - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + predicate: (not (value is not null and (value > 3))) (type: boolean) + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: int) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -236,15 +236,15 @@ STAGE PLANS: alias: predicate_fold_tb Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is null or (value >= 3)) (type: boolean) - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + predicate: (not (value is not null and (value < 3))) (type: boolean) + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: int) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 4 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -286,15 +286,15 @@ STAGE PLANS: alias: predicate_fold_tb Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is null or (value = 3)) (type: boolean) - Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + predicate: (not (value is not null and (value <> 3))) (type: boolean) + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: int) outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -334,7 +334,7 @@ STAGE PLANS: alias: predicate_fold_tb Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (value is null or (value <= 1) or (value > 3)) (type: boolean) + predicate: (not (value is not null and (value > 1) and (value <= 3))) (type: boolean) Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: int) diff --git ql/src/test/results/clientpositive/groupby_ppd.q.out ql/src/test/results/clientpositive/groupby_ppd.q.out index 515f62e..d411a0d 100644 --- ql/src/test/results/clientpositive/groupby_ppd.q.out +++ ql/src/test/results/clientpositive/groupby_ppd.q.out @@ -32,20 +32,16 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Union Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col1 + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - keys: 1 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: 1 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: 1 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE TableScan alias: c Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -58,28 +54,24 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Union Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col1 + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - keys: 1 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: 1 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: 1 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Group By Operator - keys: 1 (type: int), KEY._col1 (type: int) + keys: KEY._col0 (type: int) mode: mergepartial - outputColumnNames: _col0, _col1 + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), 1 (type: int) + expressions: _col0 (type: int), 1 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/groupby_sort_1_23.q.out ql/src/test/results/clientpositive/groupby_sort_1_23.q.out index 7ef56fc..81fe0d9 100644 --- ql/src/test/results/clientpositive/groupby_sort_1_23.q.out +++ ql/src/test/results/clientpositive/groupby_sort_1_23.q.out @@ -1387,16 +1387,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string) - outputColumnNames: _col1 + outputColumnNames: key Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: 1 (type: int), _col1 (type: string) + aggregations: count(key) + keys: key (type: string) mode: final - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 1 (type: int), UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int) + expressions: 1 (type: int), UDFToInteger(_col0) (type: int), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1482,7 +1482,7 @@ STAGE PLANS: name: default.t1 name: default.t1 Truncated Path -> Alias: - /t1 [$hdt$_0:t1] + /t1 [t1] Stage: Stage-7 Conditional Operator @@ -1708,7 +1708,7 @@ SELECT 1, key, count(1) FROM T1 GROUP BY 1, key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Output: default@outputtbl3 -POSTHOOK: Lineage: outputtbl3.cnt EXPRESSION [(t1)t1.null, ] +POSTHOOK: Lineage: outputtbl3.cnt EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl3.key1 SIMPLE [] POSTHOOK: Lineage: outputtbl3.key2 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: SELECT * FROM outputTbl3 @@ -1757,22 +1757,22 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) - null sort order: aaa - sort order: +++ - Map-reduce partition columns: _col0 (type: string), 1 (type: int), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col3 (type: bigint) + value expressions: _col2 (type: bigint) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1826,17 +1826,17 @@ STAGE PLANS: name: default.t1 name: default.t1 Truncated Path -> Alias: - /t1 [$hdt$_0:t1] + /t1 [t1] Needs Tagging: false Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1912,7 +1912,7 @@ SELECT key, 1, val, count(1) FROM T1 GROUP BY key, 1, val POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Output: default@outputtbl4 -POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t1)t1.null, ] +POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t1)t1.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl4.key3 SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ] @@ -4123,16 +4123,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -4218,7 +4218,7 @@ STAGE PLANS: name: default.t2 name: default.t2 Truncated Path -> Alias: - /t2 [$hdt$_0:t2] + /t2 [t2] Stage: Stage-7 Conditional Operator @@ -4444,7 +4444,7 @@ SELECT key, 1, val, count(1) FROM T2 GROUP BY key, 1, val POSTHOOK: type: QUERY POSTHOOK: Input: default@t2 POSTHOOK: Output: default@outputtbl4 -POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t2)t2.null, ] +POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key1 EXPRESSION [(t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl4.key3 SIMPLE [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] @@ -4502,16 +4502,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string), 2 (type: int) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), 2 (type: int), UDFToInteger(_col4) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), 2 (type: int), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -4597,7 +4597,7 @@ STAGE PLANS: name: default.t2 name: default.t2 Truncated Path -> Alias: - /t2 [$hdt$_0:t2] + /t2 [t2] Stage: Stage-7 Conditional Operator @@ -4823,7 +4823,7 @@ SELECT key, 1, val, 2, count(1) FROM T2 GROUP BY key, 1, val, 2 POSTHOOK: type: QUERY POSTHOOK: Input: default@t2 POSTHOOK: Output: default@outputtbl5 -POSTHOOK: Lineage: outputtbl5.cnt EXPRESSION [(t2)t2.null, ] +POSTHOOK: Lineage: outputtbl5.cnt EXPRESSION [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl5.key1 EXPRESSION [(t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl5.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl5.key3 SIMPLE [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] @@ -4878,16 +4878,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -5261,16 +5261,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - keys: _col0 (type: string), 2 (type: int), _col2 (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 2 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 2 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out index 2819487..5cf0ea2 100644 --- ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out +++ ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out @@ -1453,16 +1453,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string) - outputColumnNames: _col1 + outputColumnNames: key Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: 1 (type: int), _col1 (type: string) + aggregations: count(key) + keys: key (type: string) mode: final - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 1 (type: int), UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int) + expressions: 1 (type: int), UDFToInteger(_col0) (type: int), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1548,7 +1548,7 @@ STAGE PLANS: name: default.t1 name: default.t1 Truncated Path -> Alias: - /t1 [$hdt$_0:t1] + /t1 [t1] Stage: Stage-7 Conditional Operator @@ -1774,7 +1774,7 @@ SELECT 1, key, count(1) FROM T1 GROUP BY 1, key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Output: default@outputtbl3 -POSTHOOK: Lineage: outputtbl3.cnt EXPRESSION [(t1)t1.null, ] +POSTHOOK: Lineage: outputtbl3.cnt EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl3.key1 SIMPLE [] POSTHOOK: Lineage: outputtbl3.key2 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: SELECT * FROM outputTbl3 @@ -1824,22 +1824,22 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) - null sort order: aaa - sort order: +++ + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: ++ Map-reduce partition columns: rand() (type: double) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col3 (type: bigint) + value expressions: _col2 (type: bigint) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1893,14 +1893,14 @@ STAGE PLANS: name: default.t1 name: default.t1 Truncated Path -> Alias: - /t1 [$hdt$_0:t1] + /t1 [t1] Needs Tagging: false Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: partials - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1911,8 +1911,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2,_col3 - columns.types string,int,string,bigint + columns _col0,_col1,_col2 + columns.types string,string,bigint escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -1926,13 +1926,13 @@ STAGE PLANS: TableScan GatherStats: false Reduce Output Operator - key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) - null sort order: aaa - sort order: +++ - Map-reduce partition columns: _col0 (type: string), 1 (type: int), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col3 (type: bigint) + value expressions: _col2 (type: bigint) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1943,8 +1943,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2,_col3 - columns.types string,int,string,bigint + columns _col0,_col1,_col2 + columns.types string,string,bigint escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -1952,8 +1952,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col2,_col3 - columns.types string,int,string,bigint + columns _col0,_col1,_col2 + columns.types string,string,bigint escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -1963,12 +1963,12 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2044,7 +2044,7 @@ SELECT key, 1, val, count(1) FROM T1 GROUP BY key, 1, val POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Output: default@outputtbl4 -POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t1)t1.null, ] +POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t1)t1.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl4.key3 SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ] @@ -4585,16 +4585,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -4680,7 +4680,7 @@ STAGE PLANS: name: default.t2 name: default.t2 Truncated Path -> Alias: - /t2 [$hdt$_0:t2] + /t2 [t2] Stage: Stage-7 Conditional Operator @@ -4906,7 +4906,7 @@ SELECT key, 1, val, count(1) FROM T2 GROUP BY key, 1, val POSTHOOK: type: QUERY POSTHOOK: Input: default@t2 POSTHOOK: Output: default@outputtbl4 -POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t2)t2.null, ] +POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key1 EXPRESSION [(t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl4.key3 SIMPLE [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] @@ -4964,16 +4964,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string), 2 (type: int) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), 2 (type: int), UDFToInteger(_col4) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), 2 (type: int), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -5059,7 +5059,7 @@ STAGE PLANS: name: default.t2 name: default.t2 Truncated Path -> Alias: - /t2 [$hdt$_0:t2] + /t2 [t2] Stage: Stage-7 Conditional Operator @@ -5285,7 +5285,7 @@ SELECT key, 1, val, 2, count(1) FROM T2 GROUP BY key, 1, val, 2 POSTHOOK: type: QUERY POSTHOOK: Input: default@t2 POSTHOOK: Output: default@outputtbl5 -POSTHOOK: Lineage: outputtbl5.cnt EXPRESSION [(t2)t2.null, ] +POSTHOOK: Lineage: outputtbl5.cnt EXPRESSION [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl5.key1 EXPRESSION [(t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl5.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl5.key3 SIMPLE [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] @@ -5340,16 +5340,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -5723,16 +5723,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - keys: _col0 (type: string), 2 (type: int), _col2 (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 2 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 2 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/index_auto_unused.q.out ql/src/test/results/clientpositive/index_auto_unused.q.out index 81079f8..e476050 100644 --- ql/src/test/results/clientpositive/index_auto_unused.q.out +++ ql/src/test/results/clientpositive/index_auto_unused.q.out @@ -356,25 +356,37 @@ PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT * FROM srcpart WHERE ds='2008-04-09' AND hr=12 AND key < 10 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: srcpart + filterExpr: ((ds = '2008-04-09') and (12.0 = 12.0) and (UDFToDouble(key) < 10.0)) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), '2008-04-09' (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - TableScan - alias: srcpart - filterExpr: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (UDFToDouble(key) < 10.0) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string), '2008-04-09' (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - ListSink + ListSink PREHOOK: query: SELECT * FROM srcpart WHERE ds='2008-04-09' AND hr=12 AND key < 10 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/index_stale_partitioned.q.out ql/src/test/results/clientpositive/index_stale_partitioned.q.out index f2aa0e4..7d03b72 100644 --- ql/src/test/results/clientpositive/index_stale_partitioned.q.out +++ ql/src/test/results/clientpositive/index_stale_partitioned.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Processor Tree: TableScan alias: temp - filterExpr: ((UDFToDouble(key) = 86.0) and (foo = 'bar')) (type: boolean) + filterExpr: (UDFToDouble(key) = 86.0) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (UDFToDouble(key) = 86.0) (type: boolean) diff --git ql/src/test/results/clientpositive/infer_join_preds.q.out ql/src/test/results/clientpositive/infer_join_preds.q.out index 8afc905..38baed3 100644 --- ql/src/test/results/clientpositive/infer_join_preds.q.out +++ ql/src/test/results/clientpositive/infer_join_preds.q.out @@ -158,17 +158,17 @@ STAGE PLANS: alias: src1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator @@ -178,10 +178,10 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -248,19 +248,16 @@ STAGE PLANS: TableScan alias: src1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -269,10 +266,10 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 210 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -549,19 +546,16 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: false (type: boolean) - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: src1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE @@ -586,10 +580,10 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/input23.q.out ql/src/test/results/clientpositive/input23.q.out index dcb2891..e03c9e7 100644 --- ql/src/test/results/clientpositive/input23.q.out +++ ql/src/test/results/clientpositive/input23.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain extended select * from srcpart a join srcpart b where a.ds = '2008-04-08' and a.hr = '11' and b.ds = '2008-04-08' and b.hr = '14' limit 5 PREHOOK: type: QUERY @@ -144,7 +144,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from srcpart a join srcpart b where a.ds = '2008-04-08' and a.hr = '11' and b.ds = '2008-04-08' and b.hr = '14' limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@srcpart diff --git ql/src/test/results/clientpositive/input26.q.out ql/src/test/results/clientpositive/input26.q.out index 87b7081..74035c5 100644 --- ql/src/test/results/clientpositive/input26.q.out +++ ql/src/test/results/clientpositive/input26.q.out @@ -44,8 +44,8 @@ STAGE PLANS: Number of rows: 5 Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), '11' (type: string) - outputColumnNames: _col0, _col1, _col3 + expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), '11' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -60,31 +60,23 @@ STAGE PLANS: TableScan Union Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TableScan Union Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 50 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-3 Map Reduce @@ -116,8 +108,8 @@ STAGE PLANS: Number of rows: 5 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), '14' (type: string) - outputColumnNames: _col0, _col1, _col3 + expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), '14' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false diff --git ql/src/test/results/clientpositive/input6.q.out ql/src/test/results/clientpositive/input6.q.out index 5ed2767..3d1a815 100644 --- ql/src/test/results/clientpositive/input6.q.out +++ ql/src/test/results/clientpositive/input6.q.out @@ -109,7 +109,7 @@ INSERT OVERWRITE TABLE dest1 SELECT src1.key, src1.value WHERE src1.key is null POSTHOOK: type: QUERY POSTHOOK: Input: default@src1 POSTHOOK: Output: default@dest1 -POSTHOOK: Lineage: dest1.key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.key SIMPLE [] POSTHOOK: Lineage: dest1.value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT dest1.* FROM dest1 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/input8.q.out ql/src/test/results/clientpositive/input8.q.out index 03857fc..e2b6b4a 100644 --- ql/src/test/results/clientpositive/input8.q.out +++ ql/src/test/results/clientpositive/input8.q.out @@ -106,9 +106,9 @@ INSERT OVERWRITE TABLE dest1 SELECT 4 + NULL, src1.key - NULL, NULL + NULL POSTHOOK: type: QUERY POSTHOOK: Input: default@src1 POSTHOOK: Output: default@dest1 -POSTHOOK: Lineage: dest1.c1 EXPRESSION [] +POSTHOOK: Lineage: dest1.c1 SIMPLE [] POSTHOOK: Lineage: dest1.c2 EXPRESSION [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: dest1.c3 EXPRESSION [] +POSTHOOK: Lineage: dest1.c3 SIMPLE [] PREHOOK: query: SELECT dest1.* FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 diff --git ql/src/test/results/clientpositive/input_part10.q.out ql/src/test/results/clientpositive/input_part10.q.out index c8fb37e..8455bb3 100644 --- ql/src/test/results/clientpositive/input_part10.q.out +++ ql/src/test/results/clientpositive/input_part10.q.out @@ -45,6 +45,8 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 1 (type: int), 2 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 @@ -53,9 +55,10 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: int), _col1 (type: int) Reduce Operator Tree: Select Operator - expressions: 1 (type: int), 2 (type: int) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Limit diff --git ql/src/test/results/clientpositive/insert_into5.q.out ql/src/test/results/clientpositive/insert_into5.q.out index b9510b9..7b471f4 100644 --- ql/src/test/results/clientpositive/insert_into5.q.out +++ ql/src/test/results/clientpositive/insert_into5.q.out @@ -39,6 +39,8 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 1 (type: int), 'one' (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 @@ -47,9 +49,10 @@ STAGE PLANS: sort order: Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: 1 (type: int), 'one' (type: string) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 910 Basic stats: COMPLETE Column stats: COMPLETE Limit diff --git ql/src/test/results/clientpositive/insert_nonacid_from_acid.q.out ql/src/test/results/clientpositive/insert_nonacid_from_acid.q.out index f7a9853..c114654 100644 --- ql/src/test/results/clientpositive/insert_nonacid_from_acid.q.out +++ ql/src/test/results/clientpositive/insert_nonacid_from_acid.q.out @@ -53,7 +53,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@sample_06 POSTHOOK: Output: default@tab1 POSTHOOK: Lineage: tab1.age SIMPLE [(sample_06)sample_06.FieldSchema(name:age, type:int, comment:null), ] -POSTHOOK: Lineage: tab1.gpa SIMPLE [(sample_06)sample_06.FieldSchema(name:gpa, type:decimal(3,2), comment:null), ] +POSTHOOK: Lineage: tab1.gpa SIMPLE [] POSTHOOK: Lineage: tab1.name SIMPLE [(sample_06)sample_06.FieldSchema(name:name, type:varchar(50), comment:null), ] PREHOOK: query: select * from tab1 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/join38.q.out ql/src/test/results/clientpositive/join38.q.out index 7f76c5d..aca06b2 100644 --- ql/src/test/results/clientpositive/join38.q.out +++ ql/src/test/results/clientpositive/join38.q.out @@ -15,17 +15,17 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@tmp POSTHOOK: Lineage: tmp.col0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col10 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col11 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col4 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col5 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col6 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col7 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col8 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col9 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col1 SIMPLE [] +POSTHOOK: Lineage: tmp.col10 SIMPLE [] +POSTHOOK: Lineage: tmp.col11 SIMPLE [] +POSTHOOK: Lineage: tmp.col2 SIMPLE [] +POSTHOOK: Lineage: tmp.col3 SIMPLE [] +POSTHOOK: Lineage: tmp.col4 SIMPLE [] +POSTHOOK: Lineage: tmp.col5 SIMPLE [] +POSTHOOK: Lineage: tmp.col6 SIMPLE [] +POSTHOOK: Lineage: tmp.col7 SIMPLE [] +POSTHOOK: Lineage: tmp.col8 SIMPLE [] +POSTHOOK: Lineage: tmp.col9 SIMPLE [] PREHOOK: query: select * from tmp PREHOOK: type: QUERY PREHOOK: Input: default@tmp diff --git ql/src/test/results/clientpositive/join42.q.out ql/src/test/results/clientpositive/join42.q.out index 462e49e..57979b0 100644 --- ql/src/test/results/clientpositive/join42.q.out +++ ql/src/test/results/clientpositive/join42.q.out @@ -90,7 +90,7 @@ POSTHOOK: Output: default@acct POSTHOOK: Lineage: acct.acc_n EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] POSTHOOK: Lineage: acct.aid EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] POSTHOOK: Lineage: acct.brn EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] -Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: --[HIVE-10841] (WHERE col is not null) does not work sometimes for queries with many JOIN statements explain select acct.ACC_N, @@ -294,7 +294,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select acct.ACC_N, acct.brn diff --git ql/src/test/results/clientpositive/join8.q.out ql/src/test/results/clientpositive/join8.q.out index d7e7cb1..82b7357 100644 --- ql/src/test/results/clientpositive/join8.q.out +++ ql/src/test/results/clientpositive/join8.q.out @@ -153,7 +153,7 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.c3 EXPRESSION [] POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT dest1.* FROM dest1 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/join_alt_syntax.q.out ql/src/test/results/clientpositive/join_alt_syntax.q.out index 339e004..17b10bd 100644 --- ql/src/test/results/clientpositive/join_alt_syntax.q.out +++ ql/src/test/results/clientpositive/join_alt_syntax.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select p1.p_name, p2.p_name from part p1 , part p2 PREHOOK: type: QUERY @@ -230,7 +230,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select p1.p_name, p2.p_name, p3.p_name from part p1 , part p2 , part p3 where p2.p_partkey + p1.p_partkey = p1.p_partkey and p3.p_name = p2.p_name diff --git ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out index b4f1eb3..eab6178 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out @@ -172,7 +172,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 on p2.p_partkey + p1.p_partkey = p1.p_partkey and p3.p_name = p2.p_name PREHOOK: type: QUERY @@ -281,7 +281,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 on p2.p_partkey = 1 and p3.p_name = p2.p_name PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out index 3975b4f..771745f 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out @@ -176,7 +176,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 where p2.p_partkey + p1.p_partkey = p1.p_partkey and p3.p_name = p2.p_name @@ -287,7 +287,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 where p2.p_partkey = 1 and p3.p_name = p2.p_name diff --git ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out index 82d1b82..8cbc70d 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out @@ -228,7 +228,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from part p1 join part2 p2 join part3 p3 on p2_partkey + p_partkey = p1.p_partkey and p3_name = p2_name PREHOOK: type: QUERY @@ -337,7 +337,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from part p1 join part2 p2 join part3 p3 on p2_partkey = 1 and p3_name = p2_name PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out index 297154f..19a89e7 100644 --- ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out +++ ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out @@ -232,7 +232,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from part p1 join part2 p2 join part3 p3 where p2_partkey + p1.p_partkey = p1.p_partkey and p3_name = p2_name @@ -343,7 +343,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from part p1 join part2 p2 join part3 p3 where p2_partkey = 1 and p3_name = p2_name diff --git ql/src/test/results/clientpositive/join_filters.q.out ql/src/test/results/clientpositive/join_filters.q.out index 79e8b07..b909ad5 100644 --- ql/src/test/results/clientpositive/join_filters.q.out +++ ql/src/test/results/clientpositive/join_filters.q.out @@ -18,7 +18,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -774,7 +774,7 @@ NULL NULL 48 NULL NULL NULL NULL 135 NULL NULL NULL 35 UBr9lyqgsjDFvooMgQlZ9w== -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/join_nulls.q.out ql/src/test/results/clientpositive/join_nulls.q.out index b536985..2401447 100644 --- ql/src/test/results/clientpositive/join_nulls.q.out +++ ql/src/test/results/clientpositive/join_nulls.q.out @@ -18,7 +18,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -36,7 +36,7 @@ POSTHOOK: Input: default@myinput1 NULL 35 100 100 NULL 35 48 NULL NULL 35 NULL 35 -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/join_reorder.q.out ql/src/test/results/clientpositive/join_reorder.q.out index ba3c8d4..6091a5f 100644 --- ql/src/test/results/clientpositive/join_reorder.q.out +++ ql/src/test/results/clientpositive/join_reorder.q.out @@ -91,9 +91,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (UDFToDouble(_col0) + UDFToDouble(1)) (type: double) + key expressions: (UDFToDouble(_col0) + 1.0) (type: double) sort order: + - Map-reduce partition columns: (UDFToDouble(_col0) + UDFToDouble(1)) (type: double) + Map-reduce partition columns: (UDFToDouble(_col0) + 1.0) (type: double) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Reduce Operator Tree: @@ -102,7 +102,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 (UDFToDouble(_col0) + UDFToDouble(1)) (type: double) + 1 (UDFToDouble(_col0) + 1.0) (type: double) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/join_view.q.out ql/src/test/results/clientpositive/join_view.q.out index 57043fb..db68591 100644 --- ql/src/test/results/clientpositive/join_view.q.out +++ ql/src/test/results/clientpositive/join_view.q.out @@ -38,6 +38,7 @@ POSTHOOK: Input: default@invites POSTHOOK: Input: default@invites2 POSTHOOK: Output: database:default POSTHOOK: Output: default@v +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from v where ds='2011-09-01' PREHOOK: type: QUERY POSTHOOK: query: explain select * from v where ds='2011-09-01' @@ -61,9 +62,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: '2011-09-01' (type: string) - sort order: + - Map-reduce partition columns: '2011-09-01' (type: string) + sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col0 (type: string) TableScan @@ -77,9 +76,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: '2011-09-01' (type: string) - sort order: + - Map-reduce partition columns: '2011-09-01' (type: string) + sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col0 (type: int) Reduce Operator Tree: @@ -87,8 +84,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 + 1 outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator diff --git ql/src/test/results/clientpositive/lineage2.q.out ql/src/test/results/clientpositive/lineage2.q.out index a08094a..39bb6dc 100644 --- ql/src/test/results/clientpositive/lineage2.q.out +++ ql/src/test/results/clientpositive/lineage2.q.out @@ -18,7 +18,7 @@ PREHOOK: query: select * from src1 where key > 10 and value > 'val' order by key PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"773d9d0ea92e797eae292ae1eeea11ab","queryText":"select * from src1 where key > 10 and value > 'val' order by key limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2,3],"targets":[0,1],"expression":"((UDFToDouble(src1.key) > UDFToDouble(10)) and (src1.value > 'val'))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"src1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"src1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"773d9d0ea92e797eae292ae1eeea11ab","queryText":"select * from src1 where key > 10 and value > 'val' order by key limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2,3],"targets":[0,1],"expression":"((UDFToDouble(src1.key) > 10.0) and (src1.value > 'val'))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"src1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"src1.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} 146 val_146 150 val_150 213 val_213 diff --git ql/src/test/results/clientpositive/lineage3.q.out ql/src/test/results/clientpositive/lineage3.q.out index 61acf52..12ae13e 100644 --- ql/src/test/results/clientpositive/lineage3.q.out +++ ql/src/test/results/clientpositive/lineage3.q.out @@ -116,7 +116,7 @@ order by a.cbigint, a.ctinyint, b.cint, b.ctinyint limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"afd760470fc5aa6d3e8348dee03af97f","queryText":"select a.cbigint, a.ctinyint, b.cint, b.ctinyint\nfrom\n (select ctinyint, cbigint from alltypesorc\n union all\n select ctinyint, cbigint from alltypesorc) a\n inner join\n alltypesorc b\n on (a.ctinyint = b.ctinyint)\nwhere b.ctinyint < 100 and a.cbigint is not null and b.cint is not null\norder by a.cbigint, a.ctinyint, b.cint, b.ctinyint limit 5","edges":[{"sources":[4],"targets":[0],"expression":"cbigint","edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"expression":"ctinyint","edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[5],"targets":[3],"edgeType":"PROJECTION"},{"sources":[5,4],"targets":[0,1,2,3],"expression":"((alltypesorc.ctinyint < 100) and alltypesorc.cbigint is not null)","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1,2,3],"expression":"(ctinyint = alltypesorc.ctinyint)","edgeType":"PREDICATE"},{"sources":[5,6],"targets":[0,1,2,3],"expression":"((alltypesorc.ctinyint < 100) and alltypesorc.cint is not null)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.cbigint"},{"id":1,"vertexType":"COLUMN","vertexId":"a.ctinyint"},{"id":2,"vertexType":"COLUMN","vertexId":"b.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"b.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"afd760470fc5aa6d3e8348dee03af97f","queryText":"select a.cbigint, a.ctinyint, b.cint, b.ctinyint\nfrom\n (select ctinyint, cbigint from alltypesorc\n union all\n select ctinyint, cbigint from alltypesorc) a\n inner join\n alltypesorc b\n on (a.ctinyint = b.ctinyint)\nwhere b.ctinyint < 100 and a.cbigint is not null and b.cint is not null\norder by a.cbigint, a.ctinyint, b.cint, b.ctinyint limit 5","edges":[{"sources":[4],"targets":[0],"expression":"cbigint","edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"expression":"ctinyint","edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[5],"targets":[3],"edgeType":"PROJECTION"},{"sources":[5,4],"targets":[0,1,2,3],"expression":"(alltypesorc.ctinyint is not null and alltypesorc.cbigint is not null and (alltypesorc.ctinyint < 100))","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1,2,3],"expression":"(ctinyint = alltypesorc.ctinyint)","edgeType":"PREDICATE"},{"sources":[5,6],"targets":[0,1,2,3],"expression":"((alltypesorc.ctinyint < 100) and alltypesorc.cint is not null)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.cbigint"},{"id":1,"vertexType":"COLUMN","vertexId":"a.ctinyint"},{"id":2,"vertexType":"COLUMN","vertexId":"b.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"b.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"}]} -2147311592 -51 -1071480828 -51 -2147311592 -51 -1071480828 -51 -2147311592 -51 -1067683781 -51 @@ -135,7 +135,7 @@ and x.ctinyint + length(c.cstring2) < 1000 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"3a12ad24b2622a8958df12d0bdc60f8a","queryText":"select x.ctinyint, x.cint, c.cbigint-100, c.cstring1\nfrom alltypesorc c\njoin (\n select a.ctinyint ctinyint, b.cint cint\n from (select * from alltypesorc a where cboolean1=false) a\n join alltypesorc b on (a.cint = b.cbigint - 224870380)\n ) x on (x.cint = c.cint)\nwhere x.ctinyint > 10\nand x.cint < 4.5\nand x.ctinyint + length(c.cstring2) < 1000","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"expression":"(c.cbigint - UDFToLong(100))","edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[5],"targets":[0,1,2,3],"expression":"(UDFToDouble(c.cint) < 4.5)","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1,2,3],"expression":"(c.cint = c.cint)","edgeType":"PREDICATE"},{"sources":[5,6],"targets":[0,1,2,3],"expression":"((UDFToDouble(c.cint) < 4.5) and c.cbigint is not null)","edgeType":"PREDICATE"},{"sources":[6,5],"targets":[0,1,2,3],"expression":"((c.cbigint - UDFToLong(224870380)) = UDFToLong(c.cint))","edgeType":"PREDICATE"},{"sources":[8,4,5],"targets":[0,1,2,3],"expression":"((c.cboolean1 = false) and (c.ctinyint > 10) and c.cint is not null)","edgeType":"PREDICATE"},{"sources":[4,9],"targets":[0,1,2,3],"expression":"((UDFToInteger(c.ctinyint) + length(c.cstring2)) < 1000)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"x.ctinyint"},{"id":1,"vertexType":"COLUMN","vertexId":"x.cint"},{"id":2,"vertexType":"COLUMN","vertexId":"c2"},{"id":3,"vertexType":"COLUMN","vertexId":"c.cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring2"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"3a12ad24b2622a8958df12d0bdc60f8a","queryText":"select x.ctinyint, x.cint, c.cbigint-100, c.cstring1\nfrom alltypesorc c\njoin (\n select a.ctinyint ctinyint, b.cint cint\n from (select * from alltypesorc a where cboolean1=false) a\n join alltypesorc b on (a.cint = b.cbigint - 224870380)\n ) x on (x.cint = c.cint)\nwhere x.ctinyint > 10\nand x.cint < 4.5\nand x.ctinyint + length(c.cstring2) < 1000","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"expression":"(c.cbigint - 100)","edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[5],"targets":[0,1,2,3],"expression":"(UDFToDouble(c.cint) < 4.5)","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1,2,3],"expression":"(c.cint = c.cint)","edgeType":"PREDICATE"},{"sources":[5,6],"targets":[0,1,2,3],"expression":"((UDFToDouble(c.cint) < 4.5) and c.cbigint is not null)","edgeType":"PREDICATE"},{"sources":[6,5],"targets":[0,1,2,3],"expression":"((c.cbigint - 224870380) = UDFToLong(c.cint))","edgeType":"PREDICATE"},{"sources":[8,4,5],"targets":[0,1,2,3],"expression":"((c.cboolean1 = false) and (c.ctinyint > 10) and c.cint is not null)","edgeType":"PREDICATE"},{"sources":[4,9],"targets":[0,1,2,3],"expression":"((UDFToInteger(c.ctinyint) + length(c.cstring2)) < 1000)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"x.ctinyint"},{"id":1,"vertexType":"COLUMN","vertexId":"x.cint"},{"id":2,"vertexType":"COLUMN","vertexId":"c2"},{"id":3,"vertexType":"COLUMN","vertexId":"c.cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring2"}]} 11 -654374827 857266369 OEfPnHnIYueoup PREHOOK: query: select c1, x2, x3 from ( @@ -166,7 +166,7 @@ where key in (select key+18 from src1) order by key PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"8b9d63653e36ecf4dd425d3cc3de9199","queryText":"select key, value from src1\nwhere key in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"UDFToDouble(src1.key) is not null","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + UDFToDouble(18)))","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) + UDFToDouble(18)) is not null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"8b9d63653e36ecf4dd425d3cc3de9199","queryText":"select key, value from src1\nwhere key in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"UDFToDouble(src1.key) is not null","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + 18.0))","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) + 18.0) is not null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} 146 val_146 273 val_273 PREHOOK: query: select * from src1 a @@ -178,15 +178,15 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"8bf193b0658183be94e2428a79d91d10","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"((UDFToDouble(a.key) > UDFToDouble(300)) and UDFToDouble(a.key) is not null)","edgeType":"PREDICATE"},{"sources":[2,4],"targets":[0,1],"expression":"(UDFToDouble(a.key) = UDFToDouble((UDFToInteger(b.ctinyint) + 300)))","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"UDFToDouble((UDFToInteger(b.ctinyint) + 300)) is not null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"8bf193b0658183be94e2428a79d91d10","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0)","edgeType":"PREDICATE"},{"sources":[2,4],"targets":[0,1],"expression":"(UDFToDouble(a.key) = UDFToDouble((UDFToInteger(b.ctinyint) + 300)))","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"UDFToDouble((UDFToInteger(b.ctinyint) + 300)) is not null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 311 val_311 -Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[15][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select key, value from src1 where key not in (select key+18 from src1) order by key PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"9b488fe1d7cf018aad3825173808cd36","queryText":"select key, value from src1\nwhere key not in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) + UDFToDouble(18)) is null","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"(count(*) = 0)","edgeType":"PREDICATE"},{"sources":[],"targets":[0,1],"expression":"true","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + UDFToDouble(18)))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"TABLE","vertexId":"default.src1"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"9b488fe1d7cf018aad3825173808cd36","queryText":"select key, value from src1\nwhere key not in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) + 18.0) is null","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"(count(*) = 0)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + 18.0))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"TABLE","vertexId":"default.src1"}]} PREHOOK: query: select * from src1 a where not exists (select cint from alltypesorc b @@ -196,7 +196,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"53191056e05af9080a30de853e8cea9c","queryText":"select * from src1 a\nwhere not exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > UDFToDouble(300))","edgeType":"PREDICATE"},{"sources":[2,4],"targets":[0,1],"expression":"(UDFToDouble(a.key) = UDFToDouble((UDFToInteger(b.ctinyint) + 300)))","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"(UDFToInteger(b.ctinyint) + 300) is null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"53191056e05af9080a30de853e8cea9c","queryText":"select * from src1 a\nwhere not exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0)","edgeType":"PREDICATE"},{"sources":[2,4],"targets":[0,1],"expression":"(UDFToDouble(a.key) = UDFToDouble((UDFToInteger(b.ctinyint) + 300)))","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"(UDFToInteger(b.ctinyint) + 300) is null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 369 401 val_401 406 val_406 diff --git ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out index 6a9b4a1..b415ba5 100644 --- ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out @@ -73,7 +73,7 @@ SELECT key+11 FROM src WHERE key=484 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@fact_tz@ds=1/hr=2 -POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).x EXPRESSION [] #### A masked pattern was here #### PREHOOK: query: -- switch fact_daily to skewed table and point its location to /fact_tz/ds=1 alter table fact_daily skewed by (x) on (484) diff --git ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out index 8517a52..45f2481 100644 --- ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out @@ -76,7 +76,7 @@ SELECT key+11, value FROM src WHERE key=484 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@fact_tz@ds=1/hr=2 -POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).x EXPRESSION [] POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=2).y SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] #### A masked pattern was here #### PREHOOK: query: -- switch fact_daily to skewed table and point its location to /fact_tz/ds=1 @@ -517,10 +517,10 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 484 (type: int) + key expressions: _col0 (type: int) null sort order: a sort order: + - Map-reduce partition columns: 484 (type: int) + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) @@ -577,7 +577,7 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: 484 (type: int) + keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/llap/cte_mat_1.q.out ql/src/test/results/clientpositive/llap/cte_mat_1.q.out index 9ba574d..3722d9e 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_1.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_1.q.out @@ -1,3 +1,4 @@ +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -33,9 +34,7 @@ STAGE PLANS: Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '5' (type: string) - sort order: + - Map-reduce partition columns: '5' (type: string) + sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -50,9 +49,7 @@ STAGE PLANS: Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '5' (type: string) - sort order: + - Map-reduce partition columns: '5' (type: string) + sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -63,8 +60,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 '5' (type: string) - 1 '5' (type: string) + 0 + 1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: '5' (type: string) diff --git ql/src/test/results/clientpositive/llap/cte_mat_2.q.out ql/src/test/results/clientpositive/llap/cte_mat_2.q.out index 9ba574d..3722d9e 100644 --- ql/src/test/results/clientpositive/llap/cte_mat_2.q.out +++ ql/src/test/results/clientpositive/llap/cte_mat_2.q.out @@ -1,3 +1,4 @@ +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -33,9 +34,7 @@ STAGE PLANS: Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '5' (type: string) - sort order: + - Map-reduce partition columns: '5' (type: string) + sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -50,9 +49,7 @@ STAGE PLANS: Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '5' (type: string) - sort order: + - Map-reduce partition columns: '5' (type: string) + sort order: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -63,8 +60,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 '5' (type: string) - 1 '5' (type: string) + 0 + 1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: '5' (type: string) diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index 9f07718..8b04479 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -221,7 +221,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -350,7 +349,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -479,7 +477,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -608,7 +605,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -914,7 +910,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) @@ -1225,7 +1220,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) @@ -1352,7 +1346,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -1481,7 +1474,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -1606,7 +1598,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) @@ -1633,12 +1624,12 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1662,7 +1653,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1735,16 +1726,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + key expressions: (UDFToDouble(_col0) * 2.0) (type: double) sort order: + - Map-reduce partition columns: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + Map-reduce partition columns: (UDFToDouble(_col0) * 2.0) (type: double) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -1790,7 +1780,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1864,7 +1854,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) @@ -1891,9 +1880,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -1905,7 +1894,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1978,16 +1967,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + key expressions: (UDFToDouble(_col0) * 2.0) (type: double) sort order: + - Map-reduce partition columns: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + Map-reduce partition columns: (UDFToDouble(_col0) * 2.0) (type: double) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -2018,7 +2006,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -2105,16 +2093,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + key expressions: UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) sort order: + - Map-reduce partition columns: UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + Map-reduce partition columns: UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -2160,7 +2147,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + 0 UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) 1 UDFToString(_col0) (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -2250,24 +2237,23 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs Map 4 Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: '2008-04-08' (type: string) @@ -2275,9 +2261,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '2008-04-08' (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -2290,7 +2276,7 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 '2008-04-08' (type: string) - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -2319,7 +2305,7 @@ STAGE PLANS: Execution mode: llap Reduce Operator Tree: Group By Operator - keys: '2008-04-08' (type: string) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -2330,21 +2316,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: '2008-04-08' (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Dynamic Partitioning Event Operator - Target column: ds (string) - Target Input: srcpart - Partition key expr: ds - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -2357,16 +2328,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -2382,7 +2349,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- non-equi join EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY @@ -2483,7 +2450,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -2671,7 +2638,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -3328,7 +3294,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -3536,7 +3501,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -3748,7 +3712,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: ds (type: string) @@ -3786,7 +3749,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: ds (type: string) @@ -4020,7 +3982,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -4158,7 +4119,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -4619,7 +4579,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -4731,7 +4690,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) @@ -4742,7 +4700,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) input vertices: 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -4771,12 +4729,12 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -4854,7 +4812,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) @@ -4864,7 +4821,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) input vertices: 1 Map 3 @@ -4984,46 +4941,31 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 '2008-04-08' (type: string) - input vertices: - 1 Reducer 4 - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Map 2 Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: '2008-04-08' (type: string) @@ -5031,13 +4973,41 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '2008-04-08' (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 '2008-04-08' (type: string) + input vertices: + 0 Map 1 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5052,36 +5022,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: '2008-04-08' (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '2008-04-08' (type: string) - sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Dynamic Partitioning Event Operator - Target column: ds (string) - Target Input: srcpart - Partition key expr: ds - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -5094,16 +5034,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -5142,7 +5078,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -5717,7 +5652,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out index 8ed8ab4..b0024b3 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out @@ -574,7 +574,7 @@ bar baz baz baz -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: EXPLAIN SELECT agg.amount FROM agg_01 agg, dim_shops d1 @@ -651,7 +651,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT agg.amount FROM agg_01 agg, dim_shops d1 diff --git ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out index c750dc2..5bfc5e7 100644 --- ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out @@ -1297,7 +1297,7 @@ POSTHOOK: Lineage: decimal_mapjoin.cdecimal1 EXPRESSION [(alltypesorc)alltypesor POSTHOOK: Lineage: decimal_mapjoin.cdecimal2 EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1380,7 +1380,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1495,7 +1495,7 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1578,7 +1578,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint diff --git ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out index 98ba8dd..c9417f4 100644 --- ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out @@ -55,7 +55,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) @@ -186,7 +186,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -315,7 +315,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -468,7 +468,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) @@ -602,7 +602,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -734,7 +734,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) diff --git ql/src/test/results/clientpositive/llap/tez_self_join.q.out ql/src/test/results/clientpositive/llap/tez_self_join.q.out index 68f231c..48c1b09 100644 --- ql/src/test/results/clientpositive/llap/tez_self_join.q.out +++ ql/src/test/results/clientpositive/llap/tez_self_join.q.out @@ -42,6 +42,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@values__tmp__table__2 POSTHOOK: Output: default@tez_self_join2 POSTHOOK: Lineage: tez_self_join2.id1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select s.id2, s.id3 from @@ -90,9 +91,7 @@ STAGE PLANS: outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'ab' (type: string) - sort order: + - Map-reduce partition columns: 'ab' (type: string) + sort order: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: string) Execution mode: llap @@ -108,9 +107,7 @@ STAGE PLANS: Select Operator Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'ab' (type: string) - sort order: + - Map-reduce partition columns: 'ab' (type: string) + sort order: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -140,16 +137,20 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 'ab' (type: string) - 1 'ab' (type: string) + 0 + 1 outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: _col0 (type: int), _col2 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reducer 3 Execution mode: llap Reduce Operator Tree: @@ -159,10 +160,10 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2 + outputColumnNames: _col1 Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'ab' (type: string), _col2 (type: string) + expressions: 'ab' (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -179,6 +180,7 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select s.id2, s.id3 from ( diff --git ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out index 14a273b..ce86640 100644 --- ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out @@ -66,21 +66,17 @@ STAGE PLANS: alias: dummy Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 1 (type: int) - outputColumnNames: _col0 + expressions: 1 (type: int), '2014' (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), '2014' (type: string) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.partunion1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partunion1 Execution mode: llap LLAP IO: no inputs Map 3 @@ -89,21 +85,17 @@ STAGE PLANS: alias: dummy Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 2 (type: int) - outputColumnNames: _col0 + expressions: 2 (type: int), '2014' (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), '2014' (type: string) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.partunion1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partunion1 Execution mode: llap LLAP IO: no inputs Union 2 diff --git ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out index 17d5992..d02e096 100644 --- ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out @@ -55,7 +55,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) @@ -186,7 +186,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -315,7 +315,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -468,7 +468,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) @@ -602,7 +602,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -734,7 +734,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) diff --git ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index 9475487..b0f3071 100644 --- ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -1388,12 +1388,12 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1417,7 +1417,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1497,9 +1497,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + key expressions: (UDFToDouble(_col0) * 2.0) (type: double) sort order: + - Map-reduce partition columns: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + Map-reduce partition columns: (UDFToDouble(_col0) * 2.0) (type: double) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -1545,7 +1545,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1646,9 +1646,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs @@ -1660,7 +1660,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1740,9 +1740,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + key expressions: (UDFToDouble(_col0) * 2.0) (type: double) sort order: + - Map-reduce partition columns: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + Map-reduce partition columns: (UDFToDouble(_col0) * 2.0) (type: double) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -1773,7 +1773,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1867,9 +1867,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + key expressions: UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) sort order: + - Map-reduce partition columns: UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + Map-reduce partition columns: UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -1915,7 +1915,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + 0 UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) 1 UDFToString(_col0) (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1981,6 +1981,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -2005,17 +2006,13 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + sort order: + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs Map 4 @@ -2024,16 +2021,18 @@ STAGE PLANS: alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: '2008-04-08' (type: string) - mode: hash - outputColumnNames: _col0 + Select Operator Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '2008-04-08' (type: string) - sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -2043,9 +2042,9 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 '2008-04-08' (type: string) - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + 0 + 1 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -2074,32 +2073,15 @@ STAGE PLANS: Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator - keys: '2008-04-08' (type: string) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '2008-04-08' (type: string) - sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: _col0 + sort order: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Dynamic Partitioning Event Operator - Target column: ds (string) - Target Input: srcpart - Partition key expr: ds - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -2107,21 +2089,18 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -2137,7 +2116,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- non-equi join EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY @@ -2238,7 +2217,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -4372,7 +4351,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) input vertices: 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -4401,12 +4380,12 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -4494,7 +4473,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) input vertices: 1 Map 3 @@ -4599,6 +4578,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 +Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -4614,60 +4594,72 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 '2008-04-08' (type: string) - input vertices: - 1 Reducer 4 - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Map 2 Map Operator Tree: TableScan alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: '2008-04-08' (type: string) - mode: hash - outputColumnNames: _col0 + Select Operator Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '2008-04-08' (type: string) - sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 2 + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + input vertices: + 0 Map 1 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -4682,36 +4674,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - keys: '2008-04-08' (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '2008-04-08' (type: string) - sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Dynamic Partitioning Event Operator - Target column: ds (string) - Target Input: srcpart - Partition key expr: ds - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -4719,21 +4681,18 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' diff --git ql/src/test/results/clientpositive/masking_2.q.out ql/src/test/results/clientpositive/masking_2.q.out index f998cbd..ff045a9 100644 --- ql/src/test/results/clientpositive/masking_2.q.out +++ ql/src/test/results/clientpositive/masking_2.q.out @@ -192,7 +192,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and UDFToInteger(key) is not null) (type: boolean) + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) diff --git ql/src/test/results/clientpositive/mergejoin.q.out ql/src/test/results/clientpositive/mergejoin.q.out index a85fd8b..8dd86de 100644 --- ql/src/test/results/clientpositive/mergejoin.q.out +++ ql/src/test/results/clientpositive/mergejoin.q.out @@ -2692,6 +2692,7 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### NULL NULL NULL 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 +Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -2699,6 +2700,7 @@ full outer join PREHOOK: type: QUERY PREHOOK: Input: default@tab PREHOOK: Input: default@tab_part +PREHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: select * from (select * from tab where tab.key = 0)a @@ -2707,7 +2709,9 @@ full outer join POSTHOOK: type: QUERY POSTHOOK: Input: default@tab POSTHOOK: Input: default@tab_part +POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### +Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join @@ -2730,7 +2734,7 @@ NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join diff --git ql/src/test/results/clientpositive/nonblock_op_deduplicate.q.out ql/src/test/results/clientpositive/nonblock_op_deduplicate.q.out index ea25416..71ba502 100644 --- ql/src/test/results/clientpositive/nonblock_op_deduplicate.q.out +++ ql/src/test/results/clientpositive/nonblock_op_deduplicate.q.out @@ -37,7 +37,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- This test query is introduced for HIVE-4968. -- First, we do not convert the join to MapJoin. EXPLAIN @@ -140,7 +140,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT tmp4.key as key, tmp4.value as value, tmp4.count as count FROM (SELECT tmp2.key as key, tmp2.value as value, tmp3.count as count FROM (SELECT * @@ -188,7 +188,7 @@ POSTHOOK: Input: default@src1 406 val_406 25 66 val_66 25 98 val_98 25 -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: -- Then, we convert the join to MapJoin. EXPLAIN SELECT tmp4.key as key, tmp4.value as value, tmp4.count as count @@ -296,7 +296,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: SELECT tmp4.key as key, tmp4.value as value, tmp4.count as count FROM (SELECT tmp2.key as key, tmp2.value as value, tmp3.count as count FROM (SELECT * diff --git ql/src/test/results/clientpositive/orc_llap.q.out ql/src/test/results/clientpositive/orc_llap.q.out index bae69bb..658c41d 100644 --- ql/src/test/results/clientpositive/orc_llap.q.out +++ ql/src/test/results/clientpositive/orc_llap.q.out @@ -81,7 +81,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: default@cross_numbers POSTHOOK: Lineage: cross_numbers.i EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: insert into table orc_llap select ctinyint + i, csmallint + i, cint + i, cbigint + i, cfloat + i, cdouble + i, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc cross join cross_numbers @@ -121,7 +121,7 @@ POSTHOOK: Output: default@orc_llap_small POSTHOOK: Lineage: orc_llap_small.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: orc_llap_small.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: orc_llap_small.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: -- Cross join with no projection - do it on small table explain select count(1) from orc_llap_small y join orc_llap_small x @@ -202,7 +202,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: select count(1) from orc_llap_small y join orc_llap_small x PREHOOK: type: QUERY PREHOOK: Input: default@orc_llap_small @@ -657,7 +657,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@llap_temp_table #### A masked pattern was here #### -735462183586256 -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Stage-4:MAPRED' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Stage-4:MAPRED' is a cross product PREHOOK: query: -- multi-stripe test insert into table orc_llap select ctinyint + i, csmallint + i, cint + i, cbigint + i, cfloat + i, cdouble + i, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 diff --git ql/src/test/results/clientpositive/partition_multilevels.q.out ql/src/test/results/clientpositive/partition_multilevels.q.out index 948d3a0..96aa76f 100644 --- ql/src/test/results/clientpositive/partition_multilevels.q.out +++ ql/src/test/results/clientpositive/partition_multilevels.q.out @@ -991,29 +991,29 @@ STAGE PLANS: Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: level2 (type: string), level3 (type: string) - outputColumnNames: _col1, _col2 + outputColumnNames: level2, level3 Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - keys: '2222' (type: string), _col1 (type: string), _col2 (type: string) + keys: level2 (type: string), level3 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '2222' (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: '2222' (type: string), _col1 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE - value expressions: _col3 (type: bigint) + value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: '2222' (type: string), KEY._col1 (type: string), KEY._col2 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 54 Data size: 573 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '2222' (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + expressions: '2222' (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 54 Data size: 573 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1583,29 +1583,29 @@ STAGE PLANS: Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: level2 (type: string), level3 (type: string) - outputColumnNames: _col1, _col2 + outputColumnNames: level2, level3 Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - keys: '2222' (type: string), _col1 (type: string), _col2 (type: string) + keys: level2 (type: string), level3 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '2222' (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: '2222' (type: string), _col1 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE - value expressions: _col3 (type: bigint) + value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: '2222' (type: string), KEY._col1 (type: string), KEY._col2 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 54 Data size: 573 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '2222' (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: bigint) + expressions: '2222' (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 54 Data size: 573 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/pcr.q.out ql/src/test/results/clientpositive/pcr.q.out index 7222617..9daddfb 100644 --- ql/src/test/results/clientpositive/pcr.q.out +++ ql/src/test/results/clientpositive/pcr.q.out @@ -1505,7 +1505,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: 14 (type: int), KEY.reducesinkkey1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2327,23 +2327,27 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '2000-04-08' (type: string), _col3 (type: int), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4 + columns.types int,string,string,int,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Map Reduce @@ -2356,7 +2360,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string) + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2367,8 +2371,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string + columns _col0,_col1,_col2,_col3,_col4 + columns.types int,string,string,int,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -2376,8 +2380,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string + columns _col0,_col1,_col2,_col3,_col4 + columns.types int,string,string,int,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -2386,7 +2390,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), '2000-04-08' (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2627,23 +2631,27 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '2000-04-08' (type: string), _col3 (type: int), _col4 (type: string), '2000-04-09' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int,string,string,int,string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Map Reduce @@ -2656,7 +2664,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string) + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2667,8 +2675,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int,string,string,int,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -2676,8 +2684,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int,string,string,int,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -2686,7 +2694,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), '2000-04-09' (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -4845,7 +4853,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey2 (type: string) + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -5024,7 +5032,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), '11' (type: string) + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/perf/query18.q.out ql/src/test/results/clientpositive/perf/query18.q.out index a1c9da5..5bd0f1f 100644 --- ql/src/test/results/clientpositive/perf/query18.q.out +++ ql/src/test/results/clientpositive/perf/query18.q.out @@ -49,11 +49,11 @@ Stage-0 <-Map 15 [SIMPLE_EDGE] SHUFFLE [RS_28] PartitionCols:_col0 - Select Operator [SEL_20] (rows=5047 width=16) + Select Operator [SEL_20] (rows=388 width=208) Output:["_col0","_col3"] - Filter Operator [FIL_77] (rows=5047 width=16) + Filter Operator [FIL_77] (rows=388 width=208) predicate:((cd_gender = 'M') and (cd_education_status = 'College') and cd_demo_sk is not null) - TableScan [TS_18] (rows=20191 width=16) + TableScan [TS_18] (rows=1553 width=208) default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_education_status","cd_dep_count"] <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_27] diff --git ql/src/test/results/clientpositive/perf/query26.q.out ql/src/test/results/clientpositive/perf/query26.q.out index 98a85f7..6564631 100644 --- ql/src/test/results/clientpositive/perf/query26.q.out +++ ql/src/test/results/clientpositive/perf/query26.q.out @@ -38,11 +38,11 @@ Stage-0 <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_25] PartitionCols:_col0 - Select Operator [SEL_14] (rows=3106 width=13) + Select Operator [SEL_14] (rows=132 width=304) Output:["_col0"] - Filter Operator [FIL_54] (rows=3106 width=13) + Filter Operator [FIL_54] (rows=132 width=304) predicate:((cd_gender = 'F') and (cd_marital_status = 'W') and (cd_education_status = 'Primary') and cd_demo_sk is not null) - TableScan [TS_12] (rows=24850 width=13) + TableScan [TS_12] (rows=1062 width=304) default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_24] diff --git ql/src/test/results/clientpositive/perf/query27.q.out ql/src/test/results/clientpositive/perf/query27.q.out index 36302ca..d3e568d 100644 --- ql/src/test/results/clientpositive/perf/query27.q.out +++ ql/src/test/results/clientpositive/perf/query27.q.out @@ -38,11 +38,11 @@ Stage-0 <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_25] PartitionCols:_col0 - Select Operator [SEL_14] (rows=3106 width=13) + Select Operator [SEL_14] (rows=132 width=304) Output:["_col0"] - Filter Operator [FIL_54] (rows=3106 width=13) + Filter Operator [FIL_54] (rows=132 width=304) predicate:((cd_gender = 'F') and (cd_marital_status = 'D') and (cd_education_status = 'Unknown') and cd_demo_sk is not null) - TableScan [TS_12] (rows=24850 width=13) + TableScan [TS_12] (rows=1062 width=304) default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_24] diff --git ql/src/test/results/clientpositive/perf/query28.q.out ql/src/test/results/clientpositive/perf/query28.q.out index 0157928..9e956f5 100644 --- ql/src/test/results/clientpositive/perf/query28.q.out +++ ql/src/test/results/clientpositive/perf/query28.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[63][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT @@ -117,15 +117,15 @@ Stage-0 limit:100 Stage-1 Reducer 3 - File Output Operator [FS_56] - Limit [LIM_55] (rows=5 width=149) + File Output Operator [FS_51] + Limit [LIM_50] (rows=5 width=149) Number of rows:100 - Select Operator [SEL_54] (rows=5 width=149) + Select Operator [SEL_49] (rows=5 width=149) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - Merge Join Operator [MERGEJOIN_63] (rows=5 width=149) + Merge Join Operator [MERGEJOIN_58] (rows=5 width=149) Conds:(Inner),(Inner),(Inner),(Inner),(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_51] + SHUFFLE [RS_46] Group By Operator [GBY_33] (rows=1 width=136) Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] <-Map 10 [SIMPLE_EDGE] @@ -134,12 +134,12 @@ Stage-0 Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price Select Operator [SEL_30] (rows=431996724 width=88) Output:["ss_list_price"] - Filter Operator [FIL_61] (rows=431996724 width=88) + Filter Operator [FIL_56] (rows=431996724 width=88) predicate:(ss_quantity BETWEEN 11 AND 15 and (ss_list_price BETWEEN 66 AND 76 or ss_coupon_amt BETWEEN 920 AND 1920 or ss_wholesale_cost BETWEEN 4 AND 24)) TableScan [TS_28] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_52] + SHUFFLE [RS_47] Group By Operator [GBY_40] (rows=1 width=136) Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] <-Map 12 [SIMPLE_EDGE] @@ -148,12 +148,12 @@ Stage-0 Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price Select Operator [SEL_37] (rows=431996724 width=88) Output:["ss_list_price"] - Filter Operator [FIL_62] (rows=431996724 width=88) + Filter Operator [FIL_57] (rows=431996724 width=88) predicate:(ss_quantity BETWEEN 6 AND 10 and (ss_list_price BETWEEN 91 AND 101 or ss_coupon_amt BETWEEN 1430 AND 2430 or ss_wholesale_cost BETWEEN 32 AND 52)) TableScan [TS_35] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_47] + SHUFFLE [RS_42] Group By Operator [GBY_5] (rows=1 width=136) Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] <-Map 1 [SIMPLE_EDGE] @@ -162,12 +162,12 @@ Stage-0 Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price Select Operator [SEL_2] (rows=431996724 width=88) Output:["ss_list_price"] - Filter Operator [FIL_57] (rows=431996724 width=88) + Filter Operator [FIL_52] (rows=431996724 width=88) predicate:(ss_quantity BETWEEN 0 AND 5 and (ss_list_price BETWEEN 11 AND 21 or ss_coupon_amt BETWEEN 460 AND 1460 or ss_wholesale_cost BETWEEN 14 AND 34)) TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_48] + SHUFFLE [RS_43] Group By Operator [GBY_12] (rows=1 width=136) Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] <-Map 4 [SIMPLE_EDGE] @@ -176,12 +176,12 @@ Stage-0 Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price Select Operator [SEL_9] (rows=431996724 width=88) Output:["ss_list_price"] - Filter Operator [FIL_58] (rows=431996724 width=88) + Filter Operator [FIL_53] (rows=431996724 width=88) predicate:(ss_quantity BETWEEN 26 AND 30 and (ss_list_price BETWEEN 28 AND 38 or ss_coupon_amt BETWEEN 2513 AND 3513 or ss_wholesale_cost BETWEEN 42 AND 62)) TableScan [TS_7] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_49] + SHUFFLE [RS_44] Group By Operator [GBY_19] (rows=1 width=136) Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] <-Map 6 [SIMPLE_EDGE] @@ -190,12 +190,12 @@ Stage-0 Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price Select Operator [SEL_16] (rows=431996724 width=88) Output:["ss_list_price"] - Filter Operator [FIL_59] (rows=431996724 width=88) + Filter Operator [FIL_54] (rows=431996724 width=88) predicate:(ss_quantity BETWEEN 21 AND 25 and (ss_list_price BETWEEN 135 AND 145 or ss_coupon_amt BETWEEN 14180 AND 15180 or ss_wholesale_cost BETWEEN 38 AND 58)) TableScan [TS_14] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_50] + SHUFFLE [RS_45] Group By Operator [GBY_26] (rows=1 width=136) Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"] <-Map 8 [SIMPLE_EDGE] @@ -204,7 +204,7 @@ Stage-0 Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price Select Operator [SEL_23] (rows=431996724 width=88) Output:["ss_list_price"] - Filter Operator [FIL_60] (rows=431996724 width=88) + Filter Operator [FIL_55] (rows=431996724 width=88) predicate:(ss_quantity BETWEEN 16 AND 20 and (ss_list_price BETWEEN 142 AND 152 or ss_coupon_amt BETWEEN 3054 AND 4054 or ss_wholesale_cost BETWEEN 80 AND 100)) TableScan [TS_21] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] diff --git ql/src/test/results/clientpositive/perf/query31.q.out ql/src/test/results/clientpositive/perf/query31.q.out index 9d0a904..935631c 100644 --- ql/src/test/results/clientpositive/perf/query31.q.out +++ ql/src/test/results/clientpositive/perf/query31.q.out @@ -33,319 +33,305 @@ Stage-0 limit:-1 Stage-1 Reducer 7 - File Output Operator [FS_145] - Select Operator [SEL_144] (rows=347867560 width=88) + File Output Operator [FS_140] + Select Operator [SEL_139] (rows=316243230 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_143] - Select Operator [SEL_142] (rows=347867560 width=88) - Output:["_col0","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_141] (rows=347867560 width=88) - predicate:CASE WHEN ((_col7 > 0)) THEN (CASE WHEN ((_col19 > 0)) THEN (((_col23 / _col19) > (_col11 / _col7))) ELSE ((null > (_col11 / _col7))) END) ELSE (CASE WHEN ((_col19 > 0)) THEN (((_col23 / _col19) > null)) ELSE (null) END) END - Merge Join Operator [MERGEJOIN_277] (rows=695735121 width=88) - Conds:RS_138._col12=RS_139._col0(Inner),Output:["_col0","_col3","_col7","_col11","_col15","_col19","_col23"] + SHUFFLE [RS_138] + Select Operator [SEL_137] (rows=316243230 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Filter Operator [FIL_136] (rows=316243230 width=88) + predicate:CASE WHEN ((_col3 > 0)) THEN (CASE WHEN ((_col9 > 0)) THEN (((_col11 / _col9) > (_col5 / _col3))) ELSE ((null > (_col5 / _col3))) END) ELSE (CASE WHEN ((_col9 > 0)) THEN (((_col11 / _col9) > null)) ELSE (null) END) END + Merge Join Operator [MERGEJOIN_272] (rows=632486460 width=88) + Conds:RS_132._col6=RS_133._col0(Inner),Output:["_col0","_col1","_col3","_col5","_col7","_col9","_col11"] <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_139] + SHUFFLE [RS_133] PartitionCols:_col0 - Select Operator [SEL_137] (rows=87121617 width=135) - Output:["_col0","_col3"] - Group By Operator [GBY_136] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, 3, 1998 - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_135] - PartitionCols:_col0, 3, 1998 - Group By Operator [GBY_134] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col3)"],keys:_col0, 3, 1998 - Select Operator [SEL_132] (rows=174243235 width=135) - Output:["_col0","_col3"] - Merge Join Operator [MERGEJOIN_274] (rows=174243235 width=135) - Conds:RS_129._col1=RS_130._col0(Inner),Output:["_col2","_col7"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_130] - PartitionCols:_col0 - Select Operator [SEL_125] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_262] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_123] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_129] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_273] (rows=158402938 width=135) - Conds:RS_126._col0=RS_127._col0(Inner),Output:["_col1","_col2"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_126] - PartitionCols:_col0 - Select Operator [SEL_119] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_260] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_117] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_127] - PartitionCols:_col0 - Select Operator [SEL_122] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_261] (rows=18262 width=1119) - predicate:((d_qoy = 3) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_120] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] + Group By Operator [GBY_130] (rows=87121617 width=135) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 + <-Reducer 37 [SIMPLE_EDGE] + SHUFFLE [RS_129] + PartitionCols:_col0 + Group By Operator [GBY_128] (rows=174243235 width=135) + Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 + Select Operator [SEL_127] (rows=174243235 width=135) + Output:["_col7","_col2"] + Merge Join Operator [MERGEJOIN_269] (rows=174243235 width=135) + Conds:RS_124._col1=RS_125._col0(Inner),Output:["_col2","_col7"] + <-Map 40 [SIMPLE_EDGE] + SHUFFLE [RS_125] + PartitionCols:_col0 + Select Operator [SEL_120] (rows=40000000 width=1014) + Output:["_col0","_col1"] + Filter Operator [FIL_257] (rows=40000000 width=1014) + predicate:(ca_address_sk is not null and ca_county is not null) + TableScan [TS_118] (rows=40000000 width=1014) + default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] + <-Reducer 36 [SIMPLE_EDGE] + SHUFFLE [RS_124] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_268] (rows=158402938 width=135) + Conds:RS_121._col0=RS_122._col0(Inner),Output:["_col1","_col2"] + <-Map 35 [SIMPLE_EDGE] + SHUFFLE [RS_121] + PartitionCols:_col0 + Select Operator [SEL_114] (rows=144002668 width=135) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_255] (rows=144002668 width=135) + predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) + TableScan [TS_112] (rows=144002668 width=135) + default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] + <-Map 39 [SIMPLE_EDGE] + SHUFFLE [RS_122] + PartitionCols:_col0 + Select Operator [SEL_117] (rows=18262 width=1119) + Output:["_col0"] + Filter Operator [FIL_256] (rows=18262 width=1119) + predicate:((d_qoy = 3) and (d_year = 1998) and d_date_sk is not null) + TableScan [TS_115] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_138] - PartitionCols:_col12 - Select Operator [SEL_116] (rows=632486460 width=88) - Output:["_col0","_col11","_col12","_col15","_col19","_col3","_col7"] - Filter Operator [FIL_115] (rows=632486460 width=88) - predicate:CASE WHEN ((_col3 > 0)) THEN (CASE WHEN ((_col15 > 0)) THEN (((_col19 / _col15) > (_col7 / _col3))) ELSE ((null > (_col7 / _col3))) END) ELSE (CASE WHEN ((_col15 > 0)) THEN (((_col19 / _col15) > null)) ELSE (null) END) END - Select Operator [SEL_114] (rows=1264972920 width=88) - Output:["_col0","_col3","_col7","_col11","_col12","_col15","_col19"] - Merge Join Operator [MERGEJOIN_276] (rows=1264972920 width=88) - Conds:RS_109._col0=RS_110._col0(Inner),RS_109._col0=RS_111._col0(Inner),RS_111._col0=RS_112._col4(Inner),Output:["_col0","_col3","_col7","_col8","_col11","_col15","_col19"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Select Operator [SEL_41] (rows=87121617 width=135) - Output:["_col0","_col3"] - Group By Operator [GBY_40] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, 2, 1998 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_39] - PartitionCols:_col0, 2, 1998 - Group By Operator [GBY_38] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col3)"],keys:_col0, 2, 1998 - Select Operator [SEL_36] (rows=174243235 width=135) - Output:["_col0","_col3"] - Merge Join Operator [MERGEJOIN_266] (rows=174243235 width=135) - Conds:RS_33._col1=RS_34._col0(Inner),Output:["_col2","_col7"] - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_34] + SHUFFLE [RS_132] + PartitionCols:_col6 + Filter Operator [FIL_110] (rows=574987679 width=88) + predicate:CASE WHEN ((_col1 > 0)) THEN (CASE WHEN ((_col7 > 0)) THEN (((_col9 / _col7) > (_col3 / _col1))) ELSE ((null > (_col3 / _col1))) END) ELSE (CASE WHEN ((_col7 > 0)) THEN (((_col9 / _col7) > null)) ELSE (null) END) END + Select Operator [SEL_109] (rows=1149975359 width=88) + Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col9"] + Merge Join Operator [MERGEJOIN_271] (rows=1149975359 width=88) + Conds:RS_104._col0=RS_105._col0(Inner),RS_104._col0=RS_106._col0(Inner),RS_104._col0=RS_107._col0(Inner),Output:["_col0","_col1","_col3","_col5","_col6","_col7","_col9"] + <-Reducer 13 [SIMPLE_EDGE] + SHUFFLE [RS_105] + PartitionCols:_col0 + Group By Operator [GBY_38] (rows=348477374 width=88) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 + <-Reducer 12 [SIMPLE_EDGE] + SHUFFLE [RS_37] + PartitionCols:_col0 + Group By Operator [GBY_36] (rows=696954748 width=88) + Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 + Select Operator [SEL_35] (rows=696954748 width=88) + Output:["_col7","_col2"] + Merge Join Operator [MERGEJOIN_261] (rows=696954748 width=88) + Conds:RS_32._col1=RS_33._col0(Inner),Output:["_col2","_col7"] + <-Map 15 [SIMPLE_EDGE] + SHUFFLE [RS_33] + PartitionCols:_col0 + Select Operator [SEL_28] (rows=40000000 width=1014) + Output:["_col0","_col1"] + Filter Operator [FIL_245] (rows=40000000 width=1014) + predicate:(ca_address_sk is not null and ca_county is not null) + TableScan [TS_26] (rows=40000000 width=1014) + default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] + <-Reducer 11 [SIMPLE_EDGE] + SHUFFLE [RS_32] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_260] (rows=633595212 width=88) + Conds:RS_29._col0=RS_30._col0(Inner),Output:["_col1","_col2"] + <-Map 10 [SIMPLE_EDGE] + SHUFFLE [RS_29] PartitionCols:_col0 - Select Operator [SEL_29] (rows=40000000 width=1014) + Select Operator [SEL_22] (rows=575995635 width=88) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_243] (rows=575995635 width=88) + predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) + TableScan [TS_20] (rows=575995635 width=88) + default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] + <-Map 14 [SIMPLE_EDGE] + SHUFFLE [RS_30] + PartitionCols:_col0 + Select Operator [SEL_25] (rows=18262 width=1119) + Output:["_col0"] + Filter Operator [FIL_244] (rows=18262 width=1119) + predicate:((d_qoy = 1) and (d_year = 1998) and d_date_sk is not null) + TableScan [TS_23] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] + <-Reducer 19 [SIMPLE_EDGE] + SHUFFLE [RS_106] + PartitionCols:_col0 + Group By Operator [GBY_58] (rows=348477374 width=88) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 + <-Reducer 18 [SIMPLE_EDGE] + SHUFFLE [RS_57] + PartitionCols:_col0 + Group By Operator [GBY_56] (rows=696954748 width=88) + Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 + Select Operator [SEL_55] (rows=696954748 width=88) + Output:["_col7","_col2"] + Merge Join Operator [MERGEJOIN_263] (rows=696954748 width=88) + Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col2","_col7"] + <-Map 21 [SIMPLE_EDGE] + SHUFFLE [RS_53] + PartitionCols:_col0 + Select Operator [SEL_48] (rows=40000000 width=1014) + Output:["_col0","_col1"] + Filter Operator [FIL_248] (rows=40000000 width=1014) + predicate:(ca_address_sk is not null and ca_county is not null) + TableScan [TS_46] (rows=40000000 width=1014) + default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] + <-Reducer 17 [SIMPLE_EDGE] + SHUFFLE [RS_52] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_262] (rows=633595212 width=88) + Conds:RS_49._col0=RS_50._col0(Inner),Output:["_col1","_col2"] + <-Map 16 [SIMPLE_EDGE] + SHUFFLE [RS_49] + PartitionCols:_col0 + Select Operator [SEL_42] (rows=575995635 width=88) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_246] (rows=575995635 width=88) + predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) + TableScan [TS_40] (rows=575995635 width=88) + default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] + <-Map 20 [SIMPLE_EDGE] + SHUFFLE [RS_50] + PartitionCols:_col0 + Select Operator [SEL_45] (rows=18262 width=1119) + Output:["_col0"] + Filter Operator [FIL_247] (rows=18262 width=1119) + predicate:((d_qoy = 3) and (d_year = 1998) and d_date_sk is not null) + TableScan [TS_43] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] + <-Reducer 26 [SIMPLE_EDGE] + SHUFFLE [RS_107] + PartitionCols:_col0 + Merge Join Operator [MERGEJOIN_270] (rows=95833780 width=135) + Conds:RS_100._col0=RS_101._col0(Inner),Output:["_col0","_col1","_col3"] + <-Reducer 25 [SIMPLE_EDGE] + SHUFFLE [RS_100] + PartitionCols:_col0 + Group By Operator [GBY_78] (rows=87121617 width=135) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 + <-Reducer 24 [SIMPLE_EDGE] + SHUFFLE [RS_77] + PartitionCols:_col0 + Group By Operator [GBY_76] (rows=174243235 width=135) + Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 + Select Operator [SEL_75] (rows=174243235 width=135) + Output:["_col7","_col2"] + Merge Join Operator [MERGEJOIN_265] (rows=174243235 width=135) + Conds:RS_72._col1=RS_73._col0(Inner),Output:["_col2","_col7"] + <-Map 28 [SIMPLE_EDGE] + SHUFFLE [RS_73] + PartitionCols:_col0 + Select Operator [SEL_68] (rows=40000000 width=1014) Output:["_col0","_col1"] - Filter Operator [FIL_250] (rows=40000000 width=1014) + Filter Operator [FIL_251] (rows=40000000 width=1014) predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_27] (rows=40000000 width=1014) + TableScan [TS_66] (rows=40000000 width=1014) default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_33] + <-Reducer 23 [SIMPLE_EDGE] + SHUFFLE [RS_72] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_265] (rows=158402938 width=135) - Conds:RS_30._col0=RS_31._col0(Inner),Output:["_col1","_col2"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_30] + Merge Join Operator [MERGEJOIN_264] (rows=158402938 width=135) + Conds:RS_69._col0=RS_70._col0(Inner),Output:["_col1","_col2"] + <-Map 22 [SIMPLE_EDGE] + SHUFFLE [RS_69] PartitionCols:_col0 - Select Operator [SEL_23] (rows=144002668 width=135) + Select Operator [SEL_62] (rows=144002668 width=135) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_248] (rows=144002668 width=135) + Filter Operator [FIL_249] (rows=144002668 width=135) predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_21] (rows=144002668 width=135) + TableScan [TS_60] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_31] + <-Map 27 [SIMPLE_EDGE] + SHUFFLE [RS_70] PartitionCols:_col0 - Select Operator [SEL_26] (rows=18262 width=1119) + Select Operator [SEL_65] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_249] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_24] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col0 - Select Operator [SEL_62] (rows=348477374 width=88) - Output:["_col0","_col3"] - Group By Operator [GBY_61] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, 1, 1998 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0, 1, 1998 - Group By Operator [GBY_59] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col3)"],keys:_col0, 1, 1998 - Select Operator [SEL_57] (rows=696954748 width=88) - Output:["_col0","_col3"] - Merge Join Operator [MERGEJOIN_268] (rows=696954748 width=88) - Conds:RS_54._col1=RS_55._col0(Inner),Output:["_col2","_col7"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Select Operator [SEL_50] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_253] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_48] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_267] (rows=633595212 width=88) - Conds:RS_51._col0=RS_52._col0(Inner),Output:["_col1","_col2"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_251] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_42] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_252] (rows=18262 width=1119) + Filter Operator [FIL_250] (rows=18262 width=1119) predicate:((d_qoy = 1) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_45] (rows=73049 width=1119) + TableScan [TS_63] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_275] (rows=383325119 width=88) - Conds:RS_105._col0=RS_106._col0(Inner),Output:["_col3","_col4","_col7"] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0 - Select Operator [SEL_83] (rows=348477374 width=88) - Output:["_col0","_col3"] - Group By Operator [GBY_82] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, 3, 1998 - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col0, 3, 1998 - Group By Operator [GBY_80] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col3)"],keys:_col0, 3, 1998 - Select Operator [SEL_78] (rows=696954748 width=88) - Output:["_col0","_col3"] - Merge Join Operator [MERGEJOIN_270] (rows=696954748 width=88) - Conds:RS_75._col1=RS_76._col0(Inner),Output:["_col2","_col7"] - <-Map 28 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Select Operator [SEL_71] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_256] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_69] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_269] (rows=633595212 width=88) - Conds:RS_72._col0=RS_73._col0(Inner),Output:["_col1","_col2"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_72] - PartitionCols:_col0 - Select Operator [SEL_65] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_254] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_63] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col0 - Select Operator [SEL_68] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_255] (rows=18262 width=1119) - predicate:((d_qoy = 3) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_66] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 32 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col0 - Select Operator [SEL_104] (rows=348477374 width=88) - Output:["_col0","_col3"] - Group By Operator [GBY_103] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, 2, 1998 - <-Reducer 31 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col0, 2, 1998 - Group By Operator [GBY_101] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col3)"],keys:_col0, 2, 1998 - Select Operator [SEL_99] (rows=696954748 width=88) - Output:["_col0","_col3"] - Merge Join Operator [MERGEJOIN_272] (rows=696954748 width=88) - Conds:RS_96._col1=RS_97._col0(Inner),Output:["_col2","_col7"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col0 - Select Operator [SEL_92] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_259] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_90] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_271] (rows=633595212 width=88) - Conds:RS_93._col0=RS_94._col0(Inner),Output:["_col1","_col2"] - <-Map 29 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0 - Select Operator [SEL_86] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_257] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - TableScan [TS_84] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col0 - Select Operator [SEL_89] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_258] (rows=18262 width=1119) - predicate:((d_qoy = 2) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_87] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=87121617 width=135) - Output:["_col0","_col3"] - Group By Operator [GBY_19] (rows=87121617 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, 1, 1998 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0, 1, 1998 - Group By Operator [GBY_17] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col3)"],keys:_col0, 1, 1998 - Select Operator [SEL_15] (rows=174243235 width=135) - Output:["_col0","_col3"] - Merge Join Operator [MERGEJOIN_264] (rows=174243235 width=135) - Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] + <-Reducer 32 [SIMPLE_EDGE] + SHUFFLE [RS_101] + PartitionCols:_col0 + Group By Operator [GBY_98] (rows=87121617 width=135) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 + <-Reducer 31 [SIMPLE_EDGE] + SHUFFLE [RS_97] + PartitionCols:_col0 + Group By Operator [GBY_96] (rows=174243235 width=135) + Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 + Select Operator [SEL_95] (rows=174243235 width=135) + Output:["_col7","_col2"] + Merge Join Operator [MERGEJOIN_267] (rows=174243235 width=135) + Conds:RS_92._col1=RS_93._col0(Inner),Output:["_col2","_col7"] + <-Map 34 [SIMPLE_EDGE] + SHUFFLE [RS_93] PartitionCols:_col0 - Select Operator [SEL_8] (rows=40000000 width=1014) + Select Operator [SEL_88] (rows=40000000 width=1014) Output:["_col0","_col1"] - Filter Operator [FIL_247] (rows=40000000 width=1014) + Filter Operator [FIL_254] (rows=40000000 width=1014) predicate:(ca_address_sk is not null and ca_county is not null) - TableScan [TS_6] (rows=40000000 width=1014) + TableScan [TS_86] (rows=40000000 width=1014) default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_12] + <-Reducer 30 [SIMPLE_EDGE] + SHUFFLE [RS_92] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_263] (rows=158402938 width=135) - Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_9] + Merge Join Operator [MERGEJOIN_266] (rows=158402938 width=135) + Conds:RS_89._col0=RS_90._col0(Inner),Output:["_col1","_col2"] + <-Map 29 [SIMPLE_EDGE] + SHUFFLE [RS_89] PartitionCols:_col0 - Select Operator [SEL_2] (rows=144002668 width=135) + Select Operator [SEL_82] (rows=144002668 width=135) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_245] (rows=144002668 width=135) + Filter Operator [FIL_252] (rows=144002668 width=135) predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) + TableScan [TS_80] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_10] + <-Map 33 [SIMPLE_EDGE] + SHUFFLE [RS_90] PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) + Select Operator [SEL_85] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_246] (rows=18262 width=1119) - predicate:((d_qoy = 1) and (d_year = 1998) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) + Filter Operator [FIL_253] (rows=18262 width=1119) + predicate:((d_qoy = 2) and (d_year = 1998) and d_date_sk is not null) + TableScan [TS_83] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] + <-Reducer 4 [SIMPLE_EDGE] + SHUFFLE [RS_104] + PartitionCols:_col0 + Group By Operator [GBY_18] (rows=348477374 width=88) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_17] + PartitionCols:_col0 + Group By Operator [GBY_16] (rows=696954748 width=88) + Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col7 + Select Operator [SEL_15] (rows=696954748 width=88) + Output:["_col7","_col2"] + Merge Join Operator [MERGEJOIN_259] (rows=696954748 width=88) + Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7"] + <-Map 9 [SIMPLE_EDGE] + SHUFFLE [RS_13] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=40000000 width=1014) + Output:["_col0","_col1"] + Filter Operator [FIL_242] (rows=40000000 width=1014) + predicate:(ca_address_sk is not null and ca_county is not null) + TableScan [TS_6] (rows=40000000 width=1014) + default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_12] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_258] (rows=633595212 width=88) + Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_9] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=575995635 width=88) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_240] (rows=575995635 width=88) + predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) + TableScan [TS_0] (rows=575995635 width=88) + default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] + <-Map 8 [SIMPLE_EDGE] + SHUFFLE [RS_10] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=18262 width=1119) + Output:["_col0"] + Filter Operator [FIL_241] (rows=18262 width=1119) + predicate:((d_qoy = 2) and (d_year = 1998) and d_date_sk is not null) + TableScan [TS_3] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] diff --git ql/src/test/results/clientpositive/perf/query39.q.out ql/src/test/results/clientpositive/perf/query39.q.out index 04129c1..5c90a53 100644 --- ql/src/test/results/clientpositive/perf/query39.q.out +++ ql/src/test/results/clientpositive/perf/query39.q.out @@ -28,27 +28,27 @@ Stage-0 SHUFFLE [RS_60] Select Operator [SEL_59] (rows=13756683 width=15) Output:["_col0","_col1","_col3","_col4","_col5","_col6","_col8","_col9"] - Merge Join Operator [MERGEJOIN_105] (rows=13756683 width=15) - Conds:RS_56._col2, _col1=RS_57._col2, _col1(Inner),Output:["_col1","_col2","_col4","_col5","_col7","_col8","_col10","_col11"] + Merge Join Operator [MERGEJOIN_103] (rows=13756683 width=15) + Conds:RS_56._col2, _col1=RS_57._col2, _col1(Inner),Output:["_col1","_col2","_col3","_col4","_col6","_col7","_col8","_col9"] <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_57] PartitionCols:_col2, _col1 Select Operator [SEL_55] (rows=12506076 width=15) - Output:["_col1","_col2","_col4","_col5"] + Output:["_col1","_col2","_col3","_col4"] Filter Operator [FIL_54] (rows=12506076 width=15) - predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col5 / _col4) > 1.0)) END - Select Operator [SEL_97] (rows=25012152 width=15) - Output:["_col1","_col2","_col4","_col5"] - Group By Operator [GBY_53] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["avg(VALUE._col0)","stddev_samp(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, 4 + predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END + Select Operator [SEL_53] (rows=25012152 width=15) + Output:["_col1","_col2","_col3","_col4"] + Group By Operator [GBY_52] (rows=25012152 width=15) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col0, _col1, _col2, 4 - Group By Operator [GBY_51] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["avg(_col4)","stddev_samp(_col4)"],keys:_col0, _col1, _col2, 4 + SHUFFLE [RS_51] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_50] (rows=50024305 width=15) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col7, _col8, _col9 Select Operator [SEL_49] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col4"] - Merge Join Operator [MERGEJOIN_104] (rows=50024305 width=15) + Output:["_col7","_col8","_col9","_col3"] + Merge Join Operator [MERGEJOIN_102] (rows=50024305 width=15) Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col3","_col7","_col8","_col9"] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_47] @@ -62,7 +62,7 @@ Stage-0 <-Reducer 13 [SIMPLE_EDGE] SHUFFLE [RS_46] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_103] (rows=45476640 width=15) + Merge Join Operator [MERGEJOIN_101] (rows=45476640 width=15) Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col2","_col3","_col7"] <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_44] @@ -76,7 +76,7 @@ Stage-0 <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_43] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_102] (rows=41342400 width=15) + Merge Join Operator [MERGEJOIN_100] (rows=41342400 width=15) Conds:RS_40._col0=RS_41._col0(Inner),Output:["_col1","_col2","_col3"] <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_40] @@ -100,21 +100,21 @@ Stage-0 SHUFFLE [RS_56] PartitionCols:_col2, _col1 Select Operator [SEL_27] (rows=12506076 width=15) - Output:["_col1","_col2","_col4","_col5"] + Output:["_col1","_col2","_col3","_col4"] Filter Operator [FIL_26] (rows=12506076 width=15) - predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col5 / _col4) > 1.0)) END - Select Operator [SEL_98] (rows=25012152 width=15) - Output:["_col1","_col2","_col4","_col5"] - Group By Operator [GBY_25] (rows=25012152 width=15) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["avg(VALUE._col0)","stddev_samp(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, 3 + predicate:CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END + Select Operator [SEL_25] (rows=25012152 width=15) + Output:["_col1","_col2","_col3","_col4"] + Group By Operator [GBY_24] (rows=25012152 width=15) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(VALUE._col0)","avg(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2 <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0, _col1, _col2, 3 - Group By Operator [GBY_23] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["avg(_col4)","stddev_samp(_col4)"],keys:_col0, _col1, _col2, 3 + SHUFFLE [RS_23] + PartitionCols:_col0, _col1, _col2 + Group By Operator [GBY_22] (rows=50024305 width=15) + Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["stddev_samp(_col3)","avg(_col3)"],keys:_col7, _col8, _col9 Select Operator [SEL_21] (rows=50024305 width=15) - Output:["_col0","_col1","_col2","_col4"] - Merge Join Operator [MERGEJOIN_101] (rows=50024305 width=15) + Output:["_col7","_col8","_col9","_col3"] + Merge Join Operator [MERGEJOIN_99] (rows=50024305 width=15) Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col7","_col8","_col9"] <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_19] @@ -128,7 +128,7 @@ Stage-0 <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_100] (rows=45476640 width=15) + Merge Join Operator [MERGEJOIN_98] (rows=45476640 width=15) Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col7"] <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_16] @@ -142,7 +142,7 @@ Stage-0 <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_99] (rows=41342400 width=15) + Merge Join Operator [MERGEJOIN_97] (rows=41342400 width=15) Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_12] diff --git ql/src/test/results/clientpositive/perf/query42.q.out ql/src/test/results/clientpositive/perf/query42.q.out index c6c4a3c..7100bb3 100644 --- ql/src/test/results/clientpositive/perf/query42.q.out +++ ql/src/test/results/clientpositive/perf/query42.q.out @@ -15,46 +15,46 @@ Stage-0 limit:100 Stage-1 Reducer 5 - File Output Operator [FS_24] - Limit [LIM_23] (rows=100 width=88) + File Output Operator [FS_23] + Limit [LIM_22] (rows=100 width=88) Number of rows:100 - Select Operator [SEL_22] (rows=348477374 width=88) + Select Operator [SEL_21] (rows=348477374 width=88) Output:["_col0","_col1","_col2","_col3"] <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_21] - Select Operator [SEL_20] (rows=348477374 width=88) + SHUFFLE [RS_20] + Select Operator [SEL_19] (rows=348477374 width=88) Output:["_col1","_col2","_col3"] - Group By Operator [GBY_19] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:1998, KEY._col1, KEY._col2 + Group By Operator [GBY_18] (rows=348477374 width=88) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:1998, _col1, _col2 - Group By Operator [GBY_17] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col3)"],keys:1998, _col1, _col2 + SHUFFLE [RS_17] + PartitionCols:_col0, _col1 + Group By Operator [GBY_16] (rows=696954748 width=88) + Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col1","_col2","_col3"] - Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) + Output:["_col7","_col8","_col2"] + Merge Join Operator [MERGEJOIN_33] (rows=696954748 width=88) Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 Select Operator [SEL_8] (rows=231000 width=1436) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_32] (rows=231000 width=1436) + Filter Operator [FIL_31] (rows=231000 width=1436) predicate:((i_manager_id = 1) and i_item_sk is not null) TableScan [TS_6] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category_id","i_category","i_manager_id"] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_33] (rows=633595212 width=88) + Merge Join Operator [MERGEJOIN_32] (rows=633595212 width=88) Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_9] PartitionCols:_col0 Select Operator [SEL_2] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_30] (rows=575995635 width=88) + Filter Operator [FIL_29] (rows=575995635 width=88) predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] @@ -63,7 +63,7 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_5] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_31] (rows=18262 width=1119) + Filter Operator [FIL_30] (rows=18262 width=1119) predicate:((d_moy = 12) and (d_year = 1998) and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,dt,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] diff --git ql/src/test/results/clientpositive/perf/query48.q.out ql/src/test/results/clientpositive/perf/query48.q.out index 0964564..6473ad8 100644 --- ql/src/test/results/clientpositive/perf/query48.q.out +++ ql/src/test/results/clientpositive/perf/query48.q.out @@ -48,11 +48,11 @@ Stage-0 <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col0 - Select Operator [SEL_14] (rows=5047 width=16) + Select Operator [SEL_14] (rows=395 width=204) Output:["_col0"] - Filter Operator [FIL_54] (rows=5047 width=16) + Filter Operator [FIL_54] (rows=395 width=204) predicate:((cd_marital_status = 'M') and (cd_education_status = '4 yr Degree') and cd_demo_sk is not null) - TableScan [TS_12] (rows=20191 width=16) + TableScan [TS_12] (rows=1583 width=204) default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_21] diff --git ql/src/test/results/clientpositive/perf/query52.q.out ql/src/test/results/clientpositive/perf/query52.q.out index 21f3a39..c61ed30 100644 --- ql/src/test/results/clientpositive/perf/query52.q.out +++ ql/src/test/results/clientpositive/perf/query52.q.out @@ -22,17 +22,17 @@ Stage-0 Output:["_col0","_col1","_col2","_col3"] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_21] - Select Operator [SEL_20] (rows=348477374 width=88) + Select Operator [SEL_19] (rows=348477374 width=88) Output:["_col1","_col2","_col3"] - Group By Operator [GBY_19] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:1998, KEY._col1, KEY._col2 + Group By Operator [GBY_18] (rows=348477374 width=88) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:1998, _col1, _col2 - Group By Operator [GBY_17] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(_col3)"],keys:1998, _col1, _col2 + SHUFFLE [RS_17] + PartitionCols:_col0, _col1 + Group By Operator [GBY_16] (rows=696954748 width=88) + Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)"],keys:_col7, _col8 Select Operator [SEL_15] (rows=696954748 width=88) - Output:["_col1","_col2","_col3"] + Output:["_col7","_col8","_col2"] Merge Join Operator [MERGEJOIN_34] (rows=696954748 width=88) Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col7","_col8"] <-Map 7 [SIMPLE_EDGE] diff --git ql/src/test/results/clientpositive/perf/query64.q.out ql/src/test/results/clientpositive/perf/query64.q.out index 3654f42..5a5ee74 100644 --- ql/src/test/results/clientpositive/perf/query64.q.out +++ ql/src/test/results/clientpositive/perf/query64.q.out @@ -59,25 +59,25 @@ Stage-0 <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_265] Select Operator [SEL_264] (rows=331415616 width=88) - Output:["_col0","_col1","_col10","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col2","_col20","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20"] Filter Operator [FIL_263] (rows=331415616 width=88) - predicate:(_col34 <= _col15) + predicate:(_col30 <= _col13) Merge Join Operator [MERGEJOIN_658] (rows=994246850 width=88) - Conds:RS_260._col1, _col2, _col3=RS_261._col1, _col2, _col3(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col15","_col16","_col17","_col18","_col34","_col35","_col36","_col37"] + Conds:RS_260._col1, _col2, _col3=RS_261._col1, _col2, _col3(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col13","_col14","_col15","_col16","_col30","_col31","_col32","_col33"] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_260] PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_129] (rows=903860754 width=88) - Output:["_col0","_col1","_col10","_col11","_col15","_col16","_col17","_col18","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Group By Operator [GBY_128] (rows=903860754 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, 2000, KEY._col13, KEY._col14 + Select Operator [SEL_128] (rows=903860754 width=88) + Output:["_col0","_col1","_col10","_col11","_col13","_col14","_col15","_col16","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] + Group By Operator [GBY_127] (rows=903860754 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13 <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_127] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, 2000, _col13, _col14 - Group By Operator [GBY_126] (rows=1807721509 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"],aggregations:["count()","sum(_col15)","sum(_col16)","sum(_col17)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, 2000, _col13, _col14 + SHUFFLE [RS_126] + PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Group By Operator [GBY_125] (rows=1807721509 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count()","sum(_col26)","sum(_col27)","sum(_col28)"],keys:_col4, _col5, _col6, _col7, _col9, _col10, _col11, _col12, _col40, _col42, _col44, _col45, _col50, _col53 Select Operator [SEL_124] (rows=1807721509 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col13","_col14","_col15","_col16","_col17"] + Output:["_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col40","_col42","_col44","_col45","_col50","_col53","_col26","_col27","_col28"] Merge Join Operator [MERGEJOIN_656] (rows=1807721509 width=88) Conds:RS_121._col0=RS_122._col18(Inner),Output:["_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col26","_col27","_col28","_col40","_col42","_col44","_col45","_col50","_col53"] <-Reducer 13 [SIMPLE_EDGE] @@ -364,17 +364,17 @@ Stage-0 <-Reducer 45 [SIMPLE_EDGE] SHUFFLE [RS_261] PartitionCols:_col1, _col2, _col3 - Select Operator [SEL_259] (rows=903860754 width=88) - Output:["_col1","_col15","_col16","_col17","_col18","_col2","_col3"] - Group By Operator [GBY_258] (rows=903860754 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, 2001, KEY._col13, KEY._col14 + Select Operator [SEL_258] (rows=903860754 width=88) + Output:["_col1","_col13","_col14","_col15","_col16","_col2","_col3"] + Group By Operator [GBY_257] (rows=903860754 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13 <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_257] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, 2001, _col13, _col14 - Group By Operator [GBY_256] (rows=1807721509 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"],aggregations:["count()","sum(_col15)","sum(_col16)","sum(_col17)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, 2001, _col13, _col14 + SHUFFLE [RS_256] + PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Group By Operator [GBY_255] (rows=1807721509 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count()","sum(_col26)","sum(_col27)","sum(_col28)"],keys:_col4, _col5, _col6, _col7, _col9, _col10, _col11, _col12, _col40, _col42, _col44, _col45, _col50, _col53 Select Operator [SEL_254] (rows=1807721509 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col13","_col14","_col15","_col16","_col17"] + Output:["_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col40","_col42","_col44","_col45","_col50","_col53","_col26","_col27","_col28"] Merge Join Operator [MERGEJOIN_657] (rows=1807721509 width=88) Conds:RS_251._col0=RS_252._col18(Inner),Output:["_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col26","_col27","_col28","_col40","_col42","_col44","_col45","_col50","_col53"] <-Reducer 43 [SIMPLE_EDGE] diff --git ql/src/test/results/clientpositive/perf/query66.q.out ql/src/test/results/clientpositive/perf/query66.q.out index b19cc77..5eef7e7 100644 --- ql/src/test/results/clientpositive/perf/query66.q.out +++ ql/src/test/results/clientpositive/perf/query66.q.out @@ -465,168 +465,164 @@ Stage-0 <-Reducer 8 [SIMPLE_EDGE] SHUFFLE [RS_73] Select Operator [SEL_72] (rows=158120068 width=135) - Output:["_col0","_col1","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col2","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col3","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col4","_col40","_col41","_col42","_col43","_col5","_col8","_col9"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43"] Group By Operator [GBY_71] (rows=158120068 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)","sum(VALUE._col24)","sum(VALUE._col25)","sum(VALUE._col26)","sum(VALUE._col27)","sum(VALUE._col28)","sum(VALUE._col29)","sum(VALUE._col30)","sum(VALUE._col31)","sum(VALUE._col32)","sum(VALUE._col33)","sum(VALUE._col34)","sum(VALUE._col35)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, 'DIAMOND,AIRBORNE', 2002 + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)","sum(VALUE._col24)","sum(VALUE._col25)","sum(VALUE._col26)","sum(VALUE._col27)","sum(VALUE._col28)","sum(VALUE._col29)","sum(VALUE._col30)","sum(VALUE._col31)","sum(VALUE._col32)","sum(VALUE._col33)","sum(VALUE._col34)","sum(VALUE._col35)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 <-Union 7 [SIMPLE_EDGE] <-Reducer 19 [CONTAINS] Reduce Output Operator [RS_70] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, 'DIAMOND,AIRBORNE', 2002 + PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 Group By Operator [GBY_69] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43"],aggregations:["sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)","sum(_col31)","sum(_col32)","sum(_col33)","sum(_col34)","sum(_col35)","sum(_col36)","sum(_col37)","sum(_col38)","sum(_col39)","sum(_col40)","sum(_col41)","sum(_col42)","sum(_col43)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, 'DIAMOND,AIRBORNE', 2002 + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)","sum(_col31)","sum(_col32)","sum(_col33)","sum(_col34)","sum(_col35)","sum(_col36)","sum(_col37)","sum(_col38)","sum(_col39)","sum(_col40)","sum(_col41)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 Select Operator [SEL_67] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43"] - Select Operator [SEL_65] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"] - Group By Operator [GBY_64] (rows=210822976 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, 2002 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, 2002 - Group By Operator [GBY_62] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30"],aggregations:["sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, 2002 - Select Operator [SEL_60] (rows=421645953 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30"] - Merge Join Operator [MERGEJOIN_122] (rows=421645953 width=135) - Conds:RS_57._col3=RS_58._col0(Inner),Output:["_col4","_col5","_col6","_col11","_col15","_col16","_col17","_col18","_col19","_col20"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_47] (rows=27 width=1029) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_114] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_45] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_121] (rows=383314495 width=135) - Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6","_col11"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Select Operator [SEL_44] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_113] (rows=1 width=0) - predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) - TableScan [TS_42] (rows=1 width=0) - default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_120] (rows=348467716 width=135) - Conds:RS_51._col0=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col11"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col0 - Select Operator [SEL_41] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_112] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_39] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_119] (rows=316788826 width=135) - Conds:RS_48._col1=RS_49._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col1 - Select Operator [SEL_35] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_110] (rows=287989836 width=135) - predicate:(cs_warehouse_sk is not null and cs_sold_date_sk is not null and cs_sold_time_sk is not null and cs_ship_mode_sk is not null) - TableScan [TS_33] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_sold_time_sk","cs_ship_mode_sk","cs_warehouse_sk","cs_quantity","cs_ext_sales_price","cs_net_paid_inc_ship_tax"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_38] (rows=43200 width=471) - Output:["_col0"] - Filter Operator [FIL_111] (rows=43200 width=471) - predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) - TableScan [TS_36] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"] + Group By Operator [GBY_64] (rows=210822976 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 + <-Reducer 18 [SIMPLE_EDGE] + SHUFFLE [RS_63] + PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 + Group By Operator [GBY_62] (rows=421645953 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 + Select Operator [SEL_60] (rows=421645953 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"] + Merge Join Operator [MERGEJOIN_122] (rows=421645953 width=135) + Conds:RS_57._col3=RS_58._col0(Inner),Output:["_col4","_col5","_col6","_col11","_col15","_col16","_col17","_col18","_col19","_col20"] + <-Map 23 [SIMPLE_EDGE] + SHUFFLE [RS_58] + PartitionCols:_col0 + Select Operator [SEL_47] (rows=27 width=1029) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Filter Operator [FIL_114] (rows=27 width=1029) + predicate:w_warehouse_sk is not null + TableScan [TS_45] (rows=27 width=1029) + default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"] + <-Reducer 17 [SIMPLE_EDGE] + SHUFFLE [RS_57] + PartitionCols:_col3 + Merge Join Operator [MERGEJOIN_121] (rows=383314495 width=135) + Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6","_col11"] + <-Map 22 [SIMPLE_EDGE] + SHUFFLE [RS_55] + PartitionCols:_col0 + Select Operator [SEL_44] (rows=1 width=0) + Output:["_col0"] + Filter Operator [FIL_113] (rows=1 width=0) + predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) + TableScan [TS_42] (rows=1 width=0) + default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"] + <-Reducer 16 [SIMPLE_EDGE] + SHUFFLE [RS_54] + PartitionCols:_col2 + Merge Join Operator [MERGEJOIN_120] (rows=348467716 width=135) + Conds:RS_51._col0=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col11"] + <-Map 21 [SIMPLE_EDGE] + SHUFFLE [RS_52] + PartitionCols:_col0 + Select Operator [SEL_41] (rows=36524 width=1119) + Output:["_col0","_col2"] + Filter Operator [FIL_112] (rows=36524 width=1119) + predicate:((d_year = 2002) and d_date_sk is not null) + TableScan [TS_39] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + <-Reducer 15 [SIMPLE_EDGE] + SHUFFLE [RS_51] + PartitionCols:_col0 + Merge Join Operator [MERGEJOIN_119] (rows=316788826 width=135) + Conds:RS_48._col1=RS_49._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6"] + <-Map 14 [SIMPLE_EDGE] + SHUFFLE [RS_48] + PartitionCols:_col1 + Select Operator [SEL_35] (rows=287989836 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Filter Operator [FIL_110] (rows=287989836 width=135) + predicate:(cs_warehouse_sk is not null and cs_sold_date_sk is not null and cs_sold_time_sk is not null and cs_ship_mode_sk is not null) + TableScan [TS_33] (rows=287989836 width=135) + default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_sold_time_sk","cs_ship_mode_sk","cs_warehouse_sk","cs_quantity","cs_ext_sales_price","cs_net_paid_inc_ship_tax"] + <-Map 20 [SIMPLE_EDGE] + SHUFFLE [RS_49] + PartitionCols:_col0 + Select Operator [SEL_38] (rows=43200 width=471) + Output:["_col0"] + Filter Operator [FIL_111] (rows=43200 width=471) + predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) + TableScan [TS_36] (rows=86400 width=471) + default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"] <-Reducer 6 [CONTAINS] Reduce Output Operator [RS_70] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, 'DIAMOND,AIRBORNE', 2002 + PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 Group By Operator [GBY_69] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43"],aggregations:["sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)","sum(_col31)","sum(_col32)","sum(_col33)","sum(_col34)","sum(_col35)","sum(_col36)","sum(_col37)","sum(_col38)","sum(_col39)","sum(_col40)","sum(_col41)","sum(_col42)","sum(_col43)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, 'DIAMOND,AIRBORNE', 2002 + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)","sum(_col31)","sum(_col32)","sum(_col33)","sum(_col34)","sum(_col35)","sum(_col36)","sum(_col37)","sum(_col38)","sum(_col39)","sum(_col40)","sum(_col41)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 Select Operator [SEL_67] (rows=316240137 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43"] - Select Operator [SEL_32] (rows=105417161 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"] - Group By Operator [GBY_31] (rows=105417161 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, 2002 - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_30] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, 2002 - Group By Operator [GBY_29] (rows=210834322 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30"],aggregations:["sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)","sum(_col30)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, 2002 - Select Operator [SEL_27] (rows=210834322 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30"] - Merge Join Operator [MERGEJOIN_118] (rows=210834322 width=135) - Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col11","_col15","_col16","_col17","_col18","_col19","_col20"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=27 width=1029) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_109] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - TableScan [TS_12] (rows=27 width=1029) - default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_117] (rows=191667562 width=135) - Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col3","_col4","_col5","_col6","_col11"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_108] (rows=1 width=0) - predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) - TableScan [TS_9] (rows=1 width=0) - default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_116] (rows=174243235 width=135) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col11"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_107] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_6] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_115] (rows=158402938 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_105] (rows=144002668 width=135) - predicate:(ws_warehouse_sk is not null and ws_sold_date_sk is not null and ws_sold_time_sk is not null and ws_ship_mode_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_sold_time_sk","ws_ship_mode_sk","ws_warehouse_sk","ws_quantity","ws_sales_price","ws_net_paid_inc_tax"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=43200 width=471) - Output:["_col0"] - Filter Operator [FIL_106] (rows=43200 width=471) - predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) - TableScan [TS_3] (rows=86400 width=471) - default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41"] + Group By Operator [GBY_31] (rows=105417161 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)","sum(VALUE._col7)","sum(VALUE._col8)","sum(VALUE._col9)","sum(VALUE._col10)","sum(VALUE._col11)","sum(VALUE._col12)","sum(VALUE._col13)","sum(VALUE._col14)","sum(VALUE._col15)","sum(VALUE._col16)","sum(VALUE._col17)","sum(VALUE._col18)","sum(VALUE._col19)","sum(VALUE._col20)","sum(VALUE._col21)","sum(VALUE._col22)","sum(VALUE._col23)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_30] + PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 + Group By Operator [GBY_29] (rows=210834322 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)","sum(_col9)","sum(_col10)","sum(_col11)","sum(_col12)","sum(_col13)","sum(_col14)","sum(_col15)","sum(_col16)","sum(_col17)","sum(_col18)","sum(_col19)","sum(_col20)","sum(_col21)","sum(_col22)","sum(_col23)","sum(_col24)","sum(_col25)","sum(_col26)","sum(_col27)","sum(_col28)","sum(_col29)"],keys:_col0, _col1, _col2, _col3, _col4, _col5 + Select Operator [SEL_27] (rows=210834322 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29"] + Merge Join Operator [MERGEJOIN_118] (rows=210834322 width=135) + Conds:RS_24._col3=RS_25._col0(Inner),Output:["_col4","_col5","_col6","_col11","_col15","_col16","_col17","_col18","_col19","_col20"] + <-Map 13 [SIMPLE_EDGE] + SHUFFLE [RS_25] + PartitionCols:_col0 + Select Operator [SEL_14] (rows=27 width=1029) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Filter Operator [FIL_109] (rows=27 width=1029) + predicate:w_warehouse_sk is not null + TableScan [TS_12] (rows=27 width=1029) + default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"] + <-Reducer 4 [SIMPLE_EDGE] + SHUFFLE [RS_24] + PartitionCols:_col3 + Merge Join Operator [MERGEJOIN_117] (rows=191667562 width=135) + Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col3","_col4","_col5","_col6","_col11"] + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_22] + PartitionCols:_col0 + Select Operator [SEL_11] (rows=1 width=0) + Output:["_col0"] + Filter Operator [FIL_108] (rows=1 width=0) + predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) + TableScan [TS_9] (rows=1 width=0) + default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_21] + PartitionCols:_col2 + Merge Join Operator [MERGEJOIN_116] (rows=174243235 width=135) + Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6","_col11"] + <-Map 11 [SIMPLE_EDGE] + SHUFFLE [RS_19] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=36524 width=1119) + Output:["_col0","_col2"] + Filter Operator [FIL_107] (rows=36524 width=1119) + predicate:((d_year = 2002) and d_date_sk is not null) + TableScan [TS_6] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_18] + PartitionCols:_col0 + Merge Join Operator [MERGEJOIN_115] (rows=158402938 width=135) + Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_15] + PartitionCols:_col1 + Select Operator [SEL_2] (rows=144002668 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Filter Operator [FIL_105] (rows=144002668 width=135) + predicate:(ws_warehouse_sk is not null and ws_sold_date_sk is not null and ws_sold_time_sk is not null and ws_ship_mode_sk is not null) + TableScan [TS_0] (rows=144002668 width=135) + default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_sold_time_sk","ws_ship_mode_sk","ws_warehouse_sk","ws_quantity","ws_sales_price","ws_net_paid_inc_tax"] + <-Map 10 [SIMPLE_EDGE] + SHUFFLE [RS_16] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=43200 width=471) + Output:["_col0"] + Filter Operator [FIL_106] (rows=43200 width=471) + predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) + TableScan [TS_3] (rows=86400 width=471) + default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"] diff --git ql/src/test/results/clientpositive/perf/query7.q.out ql/src/test/results/clientpositive/perf/query7.q.out index 749b361..a872c84 100644 --- ql/src/test/results/clientpositive/perf/query7.q.out +++ ql/src/test/results/clientpositive/perf/query7.q.out @@ -38,11 +38,11 @@ Stage-0 <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_25] PartitionCols:_col0 - Select Operator [SEL_14] (rows=3106 width=13) + Select Operator [SEL_14] (rows=132 width=304) Output:["_col0"] - Filter Operator [FIL_54] (rows=3106 width=13) + Filter Operator [FIL_54] (rows=132 width=304) predicate:((cd_gender = 'F') and (cd_marital_status = 'W') and (cd_education_status = 'Primary') and cd_demo_sk is not null) - TableScan [TS_12] (rows=24850 width=13) + TableScan [TS_12] (rows=1062 width=304) default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status"] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_24] diff --git ql/src/test/results/clientpositive/perf/query72.q.out ql/src/test/results/clientpositive/perf/query72.q.out index 8bf7838..718129e 100644 --- ql/src/test/results/clientpositive/perf/query72.q.out +++ ql/src/test/results/clientpositive/perf/query72.q.out @@ -76,11 +76,11 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_51] PartitionCols:_col0 - Select Operator [SEL_31] (rows=32306 width=5) + Select Operator [SEL_31] (rows=1553 width=104) Output:["_col0"] - Filter Operator [FIL_131] (rows=32306 width=5) + Filter Operator [FIL_131] (rows=1553 width=104) predicate:((cd_marital_status = 'M') and cd_demo_sk is not null) - TableScan [TS_29] (rows=64612 width=5) + TableScan [TS_29] (rows=3106 width=104) default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] <-Reducer 8 [SIMPLE_EDGE] SHUFFLE [RS_50] diff --git ql/src/test/results/clientpositive/perf/query75.q.out ql/src/test/results/clientpositive/perf/query75.q.out index e44a48e..802c1af 100644 --- ql/src/test/results/clientpositive/perf/query75.q.out +++ ql/src/test/results/clientpositive/perf/query75.q.out @@ -41,367 +41,351 @@ Stage-0 <-Reducer 7 [SIMPLE_EDGE] SHUFFLE [RS_153] Select Operator [SEL_152] (rows=245965926 width=108) - Output:["_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] Filter Operator [FIL_151] (rows=245965926 width=108) - predicate:(UDFToDouble((CAST( _col5 AS decimal(17,2)) / CAST( _col12 AS decimal(17,2)))) < 0.9) + predicate:(UDFToDouble((CAST( _col4 AS decimal(17,2)) / CAST( _col10 AS decimal(17,2)))) < 0.9) Merge Join Operator [MERGEJOIN_259] (rows=737897778 width=108) - Conds:RS_148._col1, _col2, _col3, _col4=RS_149._col1, _col2, _col3, _col4(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col12","_col13"] + Conds:RS_148._col0, _col1, _col2, _col3=RS_149._col0, _col1, _col2, _col3(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col10","_col11"] <-Reducer 31 [SIMPLE_EDGE] SHUFFLE [RS_149] - PartitionCols:_col1, _col2, _col3, _col4 - Select Operator [SEL_147] (rows=670816148 width=108) - Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_146] (rows=670816148 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:2001, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Union 30 [SIMPLE_EDGE] - <-Reducer 29 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:2001, _col1, _col2, _col3, _col4 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col5)","sum(_col6)"],keys:2001, _col1, _col2, _col3, _col4 - Select Operator [SEL_142] (rows=1341632296 width=108) - Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - Select Operator [SEL_95] (rows=383314495 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_252] (rows=383314495 width=135) - Conds:RS_92._col2, _col1=RS_93._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 34 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col1, _col0 - Select Operator [SEL_85] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_232] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_83] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_92] - PartitionCols:_col2, _col1 - Merge Join Operator [MERGEJOIN_251] (rows=348467716 width=135) - Conds:RS_89._col1=RS_90._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 33 [SIMPLE_EDGE] - SHUFFLE [RS_90] + PartitionCols:_col0, _col1, _col2, _col3 + Group By Operator [GBY_146] (rows=670816148 width=108) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 + <-Union 30 [SIMPLE_EDGE] + <-Reducer 29 [CONTAINS] + Reduce Output Operator [RS_145] + PartitionCols:_col0, _col1, _col2, _col3 + Group By Operator [GBY_144] (rows=1341632296 width=108) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 + Select Operator [SEL_95] (rows=383314495 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Merge Join Operator [MERGEJOIN_252] (rows=383314495 width=135) + Conds:RS_92._col2, _col1=RS_93._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] + <-Map 34 [SIMPLE_EDGE] + SHUFFLE [RS_93] + PartitionCols:_col1, _col0 + Select Operator [SEL_85] (rows=28798881 width=106) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_232] (rows=28798881 width=106) + predicate:cr_item_sk is not null + TableScan [TS_83] (rows=28798881 width=106) + default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] + <-Reducer 28 [SIMPLE_EDGE] + SHUFFLE [RS_92] + PartitionCols:_col2, _col1 + Merge Join Operator [MERGEJOIN_251] (rows=348467716 width=135) + Conds:RS_89._col1=RS_90._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] + <-Map 33 [SIMPLE_EDGE] + SHUFFLE [RS_90] + PartitionCols:_col0 + Select Operator [SEL_82] (rows=231000 width=1436) + Output:["_col0","_col1","_col2","_col3","_col5"] + Filter Operator [FIL_231] (rows=231000 width=1436) + predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) + TableScan [TS_80] (rows=462000 width=1436) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] + <-Reducer 27 [SIMPLE_EDGE] + SHUFFLE [RS_89] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_250] (rows=316788826 width=135) + Conds:RS_86._col0=RS_87._col0(Inner),Output:["_col1","_col2","_col3","_col4"] + <-Map 26 [SIMPLE_EDGE] + SHUFFLE [RS_86] PartitionCols:_col0 - Select Operator [SEL_82] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_231] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_80] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_250] (rows=316788826 width=135) - Conds:RS_86._col0=RS_87._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_86] - PartitionCols:_col0 - Select Operator [SEL_76] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_229] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_74] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"] - <-Map 32 [SIMPLE_EDGE] - SHUFFLE [RS_87] - PartitionCols:_col0 - Select Operator [SEL_79] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_230] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_77] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 38 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:2001, _col1, _col2, _col3, _col4 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col5)","sum(_col6)"],keys:2001, _col1, _col2, _col3, _col4 - Select Operator [SEL_142] (rows=1341632296 width=108) - Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - Select Operator [SEL_117] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_255] (rows=766650239 width=88) - Conds:RS_114._col2, _col1=RS_115._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col1, _col0 - Select Operator [SEL_107] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_236] (rows=57591150 width=77) - predicate:sr_item_sk is not null - TableScan [TS_105] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_114] - PartitionCols:_col2, _col1 - Merge Join Operator [MERGEJOIN_254] (rows=696954748 width=88) - Conds:RS_111._col1=RS_112._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_112] + Select Operator [SEL_76] (rows=287989836 width=135) + Output:["_col0","_col1","_col2","_col3","_col4"] + Filter Operator [FIL_229] (rows=287989836 width=135) + predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) + TableScan [TS_74] (rows=287989836 width=135) + default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"] + <-Map 32 [SIMPLE_EDGE] + SHUFFLE [RS_87] PartitionCols:_col0 - Select Operator [SEL_104] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_235] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_102] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_111] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_253] (rows=633595212 width=88) - Conds:RS_108._col0=RS_109._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Select Operator [SEL_98] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_233] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_96] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"] - <-Map 39 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Select Operator [SEL_101] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_234] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_99] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 45 [CONTAINS] - Reduce Output Operator [RS_145] - PartitionCols:2001, _col1, _col2, _col3, _col4 - Group By Operator [GBY_144] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col5)","sum(_col6)"],keys:2001, _col1, _col2, _col3, _col4 - Select Operator [SEL_142] (rows=1341632296 width=108) - Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - Select Operator [SEL_141] (rows=191667562 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_258] (rows=191667562 width=135) - Conds:RS_138._col2, _col1=RS_139._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 48 [SIMPLE_EDGE] - SHUFFLE [RS_139] - PartitionCols:_col1, _col0 - Select Operator [SEL_131] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_240] (rows=14398467 width=92) - predicate:wr_item_sk is not null - TableScan [TS_129] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] - <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_138] - PartitionCols:_col2, _col1 - Merge Join Operator [MERGEJOIN_257] (rows=174243235 width=135) - Conds:RS_135._col1=RS_136._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_136] + Select Operator [SEL_79] (rows=36524 width=1119) + Output:["_col0"] + Filter Operator [FIL_230] (rows=36524 width=1119) + predicate:((d_year = 2001) and d_date_sk is not null) + TableScan [TS_77] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + <-Reducer 38 [CONTAINS] + Reduce Output Operator [RS_145] + PartitionCols:_col0, _col1, _col2, _col3 + Group By Operator [GBY_144] (rows=1341632296 width=108) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 + Select Operator [SEL_117] (rows=766650239 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Merge Join Operator [MERGEJOIN_255] (rows=766650239 width=88) + Conds:RS_114._col2, _col1=RS_115._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] + <-Map 41 [SIMPLE_EDGE] + SHUFFLE [RS_115] + PartitionCols:_col1, _col0 + Select Operator [SEL_107] (rows=57591150 width=77) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_236] (rows=57591150 width=77) + predicate:sr_item_sk is not null + TableScan [TS_105] (rows=57591150 width=77) + default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] + <-Reducer 37 [SIMPLE_EDGE] + SHUFFLE [RS_114] + PartitionCols:_col2, _col1 + Merge Join Operator [MERGEJOIN_254] (rows=696954748 width=88) + Conds:RS_111._col1=RS_112._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] + <-Map 40 [SIMPLE_EDGE] + SHUFFLE [RS_112] + PartitionCols:_col0 + Select Operator [SEL_104] (rows=231000 width=1436) + Output:["_col0","_col1","_col2","_col3","_col5"] + Filter Operator [FIL_235] (rows=231000 width=1436) + predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) + TableScan [TS_102] (rows=462000 width=1436) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] + <-Reducer 36 [SIMPLE_EDGE] + SHUFFLE [RS_111] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_253] (rows=633595212 width=88) + Conds:RS_108._col0=RS_109._col0(Inner),Output:["_col1","_col2","_col3","_col4"] + <-Map 35 [SIMPLE_EDGE] + SHUFFLE [RS_108] PartitionCols:_col0 - Select Operator [SEL_128] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_239] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_126] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_135] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_256] (rows=158402938 width=135) - Conds:RS_132._col0=RS_133._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_132] - PartitionCols:_col0 - Select Operator [SEL_122] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_237] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_120] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_133] - PartitionCols:_col0 - Select Operator [SEL_125] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_238] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - TableScan [TS_123] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Select Operator [SEL_98] (rows=575995635 width=88) + Output:["_col0","_col1","_col2","_col3","_col4"] + Filter Operator [FIL_233] (rows=575995635 width=88) + predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) + TableScan [TS_96] (rows=575995635 width=88) + default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"] + <-Map 39 [SIMPLE_EDGE] + SHUFFLE [RS_109] + PartitionCols:_col0 + Select Operator [SEL_101] (rows=36524 width=1119) + Output:["_col0"] + Filter Operator [FIL_234] (rows=36524 width=1119) + predicate:((d_year = 2001) and d_date_sk is not null) + TableScan [TS_99] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + <-Reducer 45 [CONTAINS] + Reduce Output Operator [RS_145] + PartitionCols:_col0, _col1, _col2, _col3 + Group By Operator [GBY_144] (rows=1341632296 width=108) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 + Select Operator [SEL_141] (rows=191667562 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Merge Join Operator [MERGEJOIN_258] (rows=191667562 width=135) + Conds:RS_138._col2, _col1=RS_139._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] + <-Map 48 [SIMPLE_EDGE] + SHUFFLE [RS_139] + PartitionCols:_col1, _col0 + Select Operator [SEL_131] (rows=14398467 width=92) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_240] (rows=14398467 width=92) + predicate:wr_item_sk is not null + TableScan [TS_129] (rows=14398467 width=92) + default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] + <-Reducer 44 [SIMPLE_EDGE] + SHUFFLE [RS_138] + PartitionCols:_col2, _col1 + Merge Join Operator [MERGEJOIN_257] (rows=174243235 width=135) + Conds:RS_135._col1=RS_136._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] + <-Map 47 [SIMPLE_EDGE] + SHUFFLE [RS_136] + PartitionCols:_col0 + Select Operator [SEL_128] (rows=231000 width=1436) + Output:["_col0","_col1","_col2","_col3","_col5"] + Filter Operator [FIL_239] (rows=231000 width=1436) + predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) + TableScan [TS_126] (rows=462000 width=1436) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] + <-Reducer 43 [SIMPLE_EDGE] + SHUFFLE [RS_135] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_256] (rows=158402938 width=135) + Conds:RS_132._col0=RS_133._col0(Inner),Output:["_col1","_col2","_col3","_col4"] + <-Map 42 [SIMPLE_EDGE] + SHUFFLE [RS_132] + PartitionCols:_col0 + Select Operator [SEL_122] (rows=144002668 width=135) + Output:["_col0","_col1","_col2","_col3","_col4"] + Filter Operator [FIL_237] (rows=144002668 width=135) + predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) + TableScan [TS_120] (rows=144002668 width=135) + default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"] + <-Map 46 [SIMPLE_EDGE] + SHUFFLE [RS_133] + PartitionCols:_col0 + Select Operator [SEL_125] (rows=36524 width=1119) + Output:["_col0"] + Filter Operator [FIL_238] (rows=36524 width=1119) + predicate:((d_year = 2001) and d_date_sk is not null) + TableScan [TS_123] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_148] - PartitionCols:_col1, _col2, _col3, _col4 - Select Operator [SEL_73] (rows=670816148 width=108) - Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_72] (rows=670816148 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:2002, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Union 5 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:2002, _col1, _col2, _col3, _col4 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col5)","sum(_col6)"],keys:2002, _col1, _col2, _col3, _col4 - Select Operator [SEL_68] (rows=1341632296 width=108) - Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - Select Operator [SEL_43] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_246] (rows=766650239 width=88) - Conds:RS_40._col2, _col1=RS_41._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col1, _col0 - Select Operator [SEL_33] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_224] (rows=57591150 width=77) - predicate:sr_item_sk is not null - TableScan [TS_31] (rows=57591150 width=77) - default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col2, _col1 - Merge Join Operator [MERGEJOIN_245] (rows=696954748 width=88) - Conds:RS_37._col1=RS_38._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 17 [SIMPLE_EDGE] - SHUFFLE [RS_38] + PartitionCols:_col0, _col1, _col2, _col3 + Group By Operator [GBY_72] (rows=670816148 width=108) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 + <-Union 5 [SIMPLE_EDGE] + <-Reducer 15 [CONTAINS] + Reduce Output Operator [RS_71] + PartitionCols:_col0, _col1, _col2, _col3 + Group By Operator [GBY_70] (rows=1341632296 width=108) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 + Select Operator [SEL_43] (rows=766650239 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Merge Join Operator [MERGEJOIN_246] (rows=766650239 width=88) + Conds:RS_40._col2, _col1=RS_41._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] + <-Map 18 [SIMPLE_EDGE] + SHUFFLE [RS_41] + PartitionCols:_col1, _col0 + Select Operator [SEL_33] (rows=57591150 width=77) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_224] (rows=57591150 width=77) + predicate:sr_item_sk is not null + TableScan [TS_31] (rows=57591150 width=77) + default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] + <-Reducer 14 [SIMPLE_EDGE] + SHUFFLE [RS_40] + PartitionCols:_col2, _col1 + Merge Join Operator [MERGEJOIN_245] (rows=696954748 width=88) + Conds:RS_37._col1=RS_38._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] + <-Map 17 [SIMPLE_EDGE] + SHUFFLE [RS_38] + PartitionCols:_col0 + Select Operator [SEL_30] (rows=231000 width=1436) + Output:["_col0","_col1","_col2","_col3","_col5"] + Filter Operator [FIL_223] (rows=231000 width=1436) + predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) + TableScan [TS_28] (rows=462000 width=1436) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] + <-Reducer 13 [SIMPLE_EDGE] + SHUFFLE [RS_37] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_244] (rows=633595212 width=88) + Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col1","_col2","_col3","_col4"] + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_34] + PartitionCols:_col0 + Select Operator [SEL_24] (rows=575995635 width=88) + Output:["_col0","_col1","_col2","_col3","_col4"] + Filter Operator [FIL_221] (rows=575995635 width=88) + predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) + TableScan [TS_22] (rows=575995635 width=88) + default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"] + <-Map 16 [SIMPLE_EDGE] + SHUFFLE [RS_35] + PartitionCols:_col0 + Select Operator [SEL_27] (rows=36524 width=1119) + Output:["_col0"] + Filter Operator [FIL_222] (rows=36524 width=1119) + predicate:((d_year = 2002) and d_date_sk is not null) + TableScan [TS_25] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + <-Reducer 22 [CONTAINS] + Reduce Output Operator [RS_71] + PartitionCols:_col0, _col1, _col2, _col3 + Group By Operator [GBY_70] (rows=1341632296 width=108) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 + Select Operator [SEL_67] (rows=191667562 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Merge Join Operator [MERGEJOIN_249] (rows=191667562 width=135) + Conds:RS_64._col2, _col1=RS_65._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] + <-Map 25 [SIMPLE_EDGE] + SHUFFLE [RS_65] + PartitionCols:_col1, _col0 + Select Operator [SEL_57] (rows=14398467 width=92) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_228] (rows=14398467 width=92) + predicate:wr_item_sk is not null + TableScan [TS_55] (rows=14398467 width=92) + default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] + <-Reducer 21 [SIMPLE_EDGE] + SHUFFLE [RS_64] + PartitionCols:_col2, _col1 + Merge Join Operator [MERGEJOIN_248] (rows=174243235 width=135) + Conds:RS_61._col1=RS_62._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] + <-Map 24 [SIMPLE_EDGE] + SHUFFLE [RS_62] + PartitionCols:_col0 + Select Operator [SEL_54] (rows=231000 width=1436) + Output:["_col0","_col1","_col2","_col3","_col5"] + Filter Operator [FIL_227] (rows=231000 width=1436) + predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) + TableScan [TS_52] (rows=462000 width=1436) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] + <-Reducer 20 [SIMPLE_EDGE] + SHUFFLE [RS_61] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_247] (rows=158402938 width=135) + Conds:RS_58._col0=RS_59._col0(Inner),Output:["_col1","_col2","_col3","_col4"] + <-Map 19 [SIMPLE_EDGE] + SHUFFLE [RS_58] + PartitionCols:_col0 + Select Operator [SEL_48] (rows=144002668 width=135) + Output:["_col0","_col1","_col2","_col3","_col4"] + Filter Operator [FIL_225] (rows=144002668 width=135) + predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) + TableScan [TS_46] (rows=144002668 width=135) + default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"] + <-Map 23 [SIMPLE_EDGE] + SHUFFLE [RS_59] PartitionCols:_col0 - Select Operator [SEL_30] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_223] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_28] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_244] (rows=633595212 width=88) - Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col0 - Select Operator [SEL_24] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_221] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - TableScan [TS_22] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"] - <-Map 16 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_27] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_222] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_25] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 22 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:2002, _col1, _col2, _col3, _col4 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col5)","sum(_col6)"],keys:2002, _col1, _col2, _col3, _col4 - Select Operator [SEL_68] (rows=1341632296 width=108) - Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - Select Operator [SEL_67] (rows=191667562 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_249] (rows=191667562 width=135) - Conds:RS_64._col2, _col1=RS_65._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col1, _col0 - Select Operator [SEL_57] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_228] (rows=14398467 width=92) - predicate:wr_item_sk is not null - TableScan [TS_55] (rows=14398467 width=92) - default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_64] - PartitionCols:_col2, _col1 - Merge Join Operator [MERGEJOIN_248] (rows=174243235 width=135) - Conds:RS_61._col1=RS_62._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 24 [SIMPLE_EDGE] - SHUFFLE [RS_62] + Select Operator [SEL_51] (rows=36524 width=1119) + Output:["_col0"] + Filter Operator [FIL_226] (rows=36524 width=1119) + predicate:((d_year = 2002) and d_date_sk is not null) + TableScan [TS_49] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + <-Reducer 4 [CONTAINS] + Reduce Output Operator [RS_71] + PartitionCols:_col0, _col1, _col2, _col3 + Group By Operator [GBY_70] (rows=1341632296 width=108) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 + Select Operator [SEL_21] (rows=383314495 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Merge Join Operator [MERGEJOIN_243] (rows=383314495 width=135) + Conds:RS_18._col2, _col1=RS_19._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] + <-Map 11 [SIMPLE_EDGE] + SHUFFLE [RS_19] + PartitionCols:_col1, _col0 + Select Operator [SEL_11] (rows=28798881 width=106) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_220] (rows=28798881 width=106) + predicate:cr_item_sk is not null + TableScan [TS_9] (rows=28798881 width=106) + default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_18] + PartitionCols:_col2, _col1 + Merge Join Operator [MERGEJOIN_242] (rows=348467716 width=135) + Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] + <-Map 10 [SIMPLE_EDGE] + SHUFFLE [RS_16] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=231000 width=1436) + Output:["_col0","_col1","_col2","_col3","_col5"] + Filter Operator [FIL_219] (rows=231000 width=1436) + predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null) + TableScan [TS_6] (rows=462000 width=1436) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_15] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_241] (rows=316788826 width=135) + Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_12] PartitionCols:_col0 - Select Operator [SEL_54] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_227] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_52] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_247] (rows=158402938 width=135) - Conds:RS_58._col0=RS_59._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_58] - PartitionCols:_col0 - Select Operator [SEL_48] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_225] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - TableScan [TS_46] (rows=144002668 width=135) - default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"] - <-Map 23 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0 - Select Operator [SEL_51] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_226] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_49] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 4 [CONTAINS] - Reduce Output Operator [RS_71] - PartitionCols:2002, _col1, _col2, _col3, _col4 - Group By Operator [GBY_70] (rows=1341632296 width=108) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col5)","sum(_col6)"],keys:2002, _col1, _col2, _col3, _col4 - Select Operator [SEL_68] (rows=1341632296 width=108) - Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - Select Operator [SEL_21] (rows=383314495 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Merge Join Operator [MERGEJOIN_243] (rows=383314495 width=135) - Conds:RS_18._col2, _col1=RS_19._col1, _col0(Left Outer),Output:["_col3","_col4","_col8","_col9","_col10","_col12","_col15","_col16"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col1, _col0 - Select Operator [SEL_11] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_220] (rows=28798881 width=106) - predicate:cr_item_sk is not null - TableScan [TS_9] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2, _col1 - Merge Join Operator [MERGEJOIN_242] (rows=348467716 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col8","_col9","_col10","_col12"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_16] + Select Operator [SEL_2] (rows=287989836 width=135) + Output:["_col0","_col1","_col2","_col3","_col4"] + Filter Operator [FIL_217] (rows=287989836 width=135) + predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) + TableScan [TS_0] (rows=287989836 width=135) + default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"] + <-Map 9 [SIMPLE_EDGE] + SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_219] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - TableScan [TS_6] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_241] (rows=316788826 width=135) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_217] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_218] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] + Select Operator [SEL_5] (rows=36524 width=1119) + Output:["_col0"] + Filter Operator [FIL_218] (rows=36524 width=1119) + predicate:((d_year = 2002) and d_date_sk is not null) + TableScan [TS_3] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] diff --git ql/src/test/results/clientpositive/perf/query88.q.out ql/src/test/results/clientpositive/perf/query88.q.out index e3f69d3..9981af5 100644 --- ql/src/test/results/clientpositive/perf/query88.q.out +++ ql/src/test/results/clientpositive/perf/query88.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[354][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[347][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5, $hdt$_6, $hdt$_7]] in Stage 'Reducer 6' is a cross product PREHOOK: query: explain select * from @@ -225,55 +225,55 @@ Stage-0 limit:-1 Stage-1 Reducer 6 - File Output Operator [FS_225] - Select Operator [SEL_224] (rows=7 width=8) + File Output Operator [FS_218] + Select Operator [SEL_217] (rows=7 width=8) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_354] (rows=7 width=8) + Merge Join Operator [MERGEJOIN_347] (rows=7 width=8) Conds:(Inner),(Inner),(Inner),(Inner),(Inner),(Inner),(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_216] + SHUFFLE [RS_209] Group By Operator [GBY_50] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 13 [SIMPLE_EDGE] SHUFFLE [RS_49] Group By Operator [GBY_48] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_335] (rows=766650239 width=88) + Merge Join Operator [MERGEJOIN_328] (rows=766650239 width=88) Conds:RS_44._col2=RS_45._col0(Inner) <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_45] PartitionCols:_col0 Select Operator [SEL_37] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_305] (rows=852 width=1910) + Filter Operator [FIL_298] (rows=852 width=1910) predicate:((s_store_name = 'ese') and s_store_sk is not null) TableScan [TS_35] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_44] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_334] (rows=696954748 width=88) + Merge Join Operator [MERGEJOIN_327] (rows=696954748 width=88) Conds:RS_41._col1=RS_42._col0(Inner),Output:["_col2"] <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_42] PartitionCols:_col0 Select Operator [SEL_34] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_304] (rows=3600 width=107) + Filter Operator [FIL_297] (rows=3600 width=107) predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) TableScan [TS_32] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_41] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_333] (rows=633595212 width=88) + Merge Join Operator [MERGEJOIN_326] (rows=633595212 width=88) Conds:RS_38._col0=RS_39._col0(Inner),Output:["_col1","_col2"] <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:_col0 Select Operator [SEL_28] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_302] (rows=575995635 width=88) + Filter Operator [FIL_295] (rows=575995635 width=88) predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) TableScan [TS_26] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] @@ -282,54 +282,54 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_31] (rows=14400 width=471) Output:["_col0"] - Filter Operator [FIL_303] (rows=14400 width=471) + Filter Operator [FIL_296] (rows=14400 width=471) predicate:((t_hour = 12) and (t_minute < 30) and t_time_sk is not null) TableScan [TS_29] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_217] + SHUFFLE [RS_210] Group By Operator [GBY_76] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 21 [SIMPLE_EDGE] SHUFFLE [RS_75] Group By Operator [GBY_74] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_338] (rows=766650239 width=88) + Merge Join Operator [MERGEJOIN_331] (rows=766650239 width=88) Conds:RS_70._col2=RS_71._col0(Inner) <-Map 25 [SIMPLE_EDGE] SHUFFLE [RS_71] PartitionCols:_col0 Select Operator [SEL_63] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_309] (rows=852 width=1910) + Filter Operator [FIL_302] (rows=852 width=1910) predicate:((s_store_name = 'ese') and s_store_sk is not null) TableScan [TS_61] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_70] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_337] (rows=696954748 width=88) + Merge Join Operator [MERGEJOIN_330] (rows=696954748 width=88) Conds:RS_67._col1=RS_68._col0(Inner),Output:["_col2"] <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_68] PartitionCols:_col0 Select Operator [SEL_60] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_308] (rows=3600 width=107) + Filter Operator [FIL_301] (rows=3600 width=107) predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) TableScan [TS_58] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 19 [SIMPLE_EDGE] SHUFFLE [RS_67] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_336] (rows=633595212 width=88) + Merge Join Operator [MERGEJOIN_329] (rows=633595212 width=88) Conds:RS_64._col0=RS_65._col0(Inner),Output:["_col1","_col2"] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_64] PartitionCols:_col0 Select Operator [SEL_54] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_306] (rows=575995635 width=88) + Filter Operator [FIL_299] (rows=575995635 width=88) predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) TableScan [TS_52] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] @@ -338,54 +338,54 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_57] (rows=14400 width=471) Output:["_col0"] - Filter Operator [FIL_307] (rows=14400 width=471) + Filter Operator [FIL_300] (rows=14400 width=471) predicate:((t_hour = 11) and (t_minute >= 30) and t_time_sk is not null) TableScan [TS_55] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_218] + SHUFFLE [RS_211] Group By Operator [GBY_102] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 29 [SIMPLE_EDGE] SHUFFLE [RS_101] Group By Operator [GBY_100] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_341] (rows=766650239 width=88) + Merge Join Operator [MERGEJOIN_334] (rows=766650239 width=88) Conds:RS_96._col2=RS_97._col0(Inner) <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_97] PartitionCols:_col0 Select Operator [SEL_89] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_313] (rows=852 width=1910) + Filter Operator [FIL_306] (rows=852 width=1910) predicate:((s_store_name = 'ese') and s_store_sk is not null) TableScan [TS_87] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 28 [SIMPLE_EDGE] SHUFFLE [RS_96] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_340] (rows=696954748 width=88) + Merge Join Operator [MERGEJOIN_333] (rows=696954748 width=88) Conds:RS_93._col1=RS_94._col0(Inner),Output:["_col2"] <-Map 32 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0 Select Operator [SEL_86] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_312] (rows=3600 width=107) + Filter Operator [FIL_305] (rows=3600 width=107) predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) TableScan [TS_84] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 27 [SIMPLE_EDGE] SHUFFLE [RS_93] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_339] (rows=633595212 width=88) + Merge Join Operator [MERGEJOIN_332] (rows=633595212 width=88) Conds:RS_90._col0=RS_91._col0(Inner),Output:["_col1","_col2"] <-Map 26 [SIMPLE_EDGE] SHUFFLE [RS_90] PartitionCols:_col0 Select Operator [SEL_80] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_310] (rows=575995635 width=88) + Filter Operator [FIL_303] (rows=575995635 width=88) predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) TableScan [TS_78] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] @@ -394,54 +394,54 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_83] (rows=14400 width=471) Output:["_col0"] - Filter Operator [FIL_311] (rows=14400 width=471) + Filter Operator [FIL_304] (rows=14400 width=471) predicate:((t_hour = 11) and (t_minute < 30) and t_time_sk is not null) TableScan [TS_81] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_219] + SHUFFLE [RS_212] Group By Operator [GBY_128] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 37 [SIMPLE_EDGE] SHUFFLE [RS_127] Group By Operator [GBY_126] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_344] (rows=766650239 width=88) + Merge Join Operator [MERGEJOIN_337] (rows=766650239 width=88) Conds:RS_122._col2=RS_123._col0(Inner) <-Map 41 [SIMPLE_EDGE] SHUFFLE [RS_123] PartitionCols:_col0 Select Operator [SEL_115] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_317] (rows=852 width=1910) + Filter Operator [FIL_310] (rows=852 width=1910) predicate:((s_store_name = 'ese') and s_store_sk is not null) TableScan [TS_113] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 36 [SIMPLE_EDGE] SHUFFLE [RS_122] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_343] (rows=696954748 width=88) + Merge Join Operator [MERGEJOIN_336] (rows=696954748 width=88) Conds:RS_119._col1=RS_120._col0(Inner),Output:["_col2"] <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_120] PartitionCols:_col0 Select Operator [SEL_112] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_316] (rows=3600 width=107) + Filter Operator [FIL_309] (rows=3600 width=107) predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) TableScan [TS_110] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 35 [SIMPLE_EDGE] SHUFFLE [RS_119] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_342] (rows=633595212 width=88) + Merge Join Operator [MERGEJOIN_335] (rows=633595212 width=88) Conds:RS_116._col0=RS_117._col0(Inner),Output:["_col1","_col2"] <-Map 34 [SIMPLE_EDGE] SHUFFLE [RS_116] PartitionCols:_col0 Select Operator [SEL_106] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_314] (rows=575995635 width=88) + Filter Operator [FIL_307] (rows=575995635 width=88) predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) TableScan [TS_104] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] @@ -450,54 +450,54 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_109] (rows=14400 width=471) Output:["_col0"] - Filter Operator [FIL_315] (rows=14400 width=471) + Filter Operator [FIL_308] (rows=14400 width=471) predicate:((t_hour = 10) and (t_minute >= 30) and t_time_sk is not null) TableScan [TS_107] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] <-Reducer 46 [SIMPLE_EDGE] - SHUFFLE [RS_220] + SHUFFLE [RS_213] Group By Operator [GBY_154] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 45 [SIMPLE_EDGE] SHUFFLE [RS_153] Group By Operator [GBY_152] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_347] (rows=766650239 width=88) + Merge Join Operator [MERGEJOIN_340] (rows=766650239 width=88) Conds:RS_148._col2=RS_149._col0(Inner) <-Map 49 [SIMPLE_EDGE] SHUFFLE [RS_149] PartitionCols:_col0 Select Operator [SEL_141] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_321] (rows=852 width=1910) + Filter Operator [FIL_314] (rows=852 width=1910) predicate:((s_store_name = 'ese') and s_store_sk is not null) TableScan [TS_139] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 44 [SIMPLE_EDGE] SHUFFLE [RS_148] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_346] (rows=696954748 width=88) + Merge Join Operator [MERGEJOIN_339] (rows=696954748 width=88) Conds:RS_145._col1=RS_146._col0(Inner),Output:["_col2"] <-Map 48 [SIMPLE_EDGE] SHUFFLE [RS_146] PartitionCols:_col0 Select Operator [SEL_138] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_320] (rows=3600 width=107) + Filter Operator [FIL_313] (rows=3600 width=107) predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) TableScan [TS_136] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 43 [SIMPLE_EDGE] SHUFFLE [RS_145] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_345] (rows=633595212 width=88) + Merge Join Operator [MERGEJOIN_338] (rows=633595212 width=88) Conds:RS_142._col0=RS_143._col0(Inner),Output:["_col1","_col2"] <-Map 42 [SIMPLE_EDGE] SHUFFLE [RS_142] PartitionCols:_col0 Select Operator [SEL_132] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_318] (rows=575995635 width=88) + Filter Operator [FIL_311] (rows=575995635 width=88) predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) TableScan [TS_130] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] @@ -506,54 +506,54 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_135] (rows=14400 width=471) Output:["_col0"] - Filter Operator [FIL_319] (rows=14400 width=471) + Filter Operator [FIL_312] (rows=14400 width=471) predicate:((t_hour = 10) and (t_minute < 30) and t_time_sk is not null) TableScan [TS_133] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_215] + SHUFFLE [RS_208] Group By Operator [GBY_24] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_23] Group By Operator [GBY_22] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_332] (rows=766650239 width=88) + Merge Join Operator [MERGEJOIN_325] (rows=766650239 width=88) Conds:RS_18._col2=RS_19._col0(Inner) <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 Select Operator [SEL_11] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_301] (rows=852 width=1910) + Filter Operator [FIL_294] (rows=852 width=1910) predicate:((s_store_name = 'ese') and s_store_sk is not null) TableScan [TS_9] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_331] (rows=696954748 width=88) + Merge Join Operator [MERGEJOIN_324] (rows=696954748 width=88) Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2"] <-Map 8 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 Select Operator [SEL_8] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_300] (rows=3600 width=107) + Filter Operator [FIL_293] (rows=3600 width=107) predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) TableScan [TS_6] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_330] (rows=633595212 width=88) + Merge Join Operator [MERGEJOIN_323] (rows=633595212 width=88) Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col0 Select Operator [SEL_2] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_298] (rows=575995635 width=88) + Filter Operator [FIL_291] (rows=575995635 width=88) predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] @@ -562,54 +562,54 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_5] (rows=14400 width=471) Output:["_col0"] - Filter Operator [FIL_299] (rows=14400 width=471) + Filter Operator [FIL_292] (rows=14400 width=471) predicate:((t_hour = 8) and (t_minute >= 30) and t_time_sk is not null) TableScan [TS_3] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] <-Reducer 54 [SIMPLE_EDGE] - SHUFFLE [RS_221] + SHUFFLE [RS_214] Group By Operator [GBY_180] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 53 [SIMPLE_EDGE] SHUFFLE [RS_179] Group By Operator [GBY_178] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_350] (rows=766650239 width=88) + Merge Join Operator [MERGEJOIN_343] (rows=766650239 width=88) Conds:RS_174._col2=RS_175._col0(Inner) <-Map 57 [SIMPLE_EDGE] SHUFFLE [RS_175] PartitionCols:_col0 Select Operator [SEL_167] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_325] (rows=852 width=1910) + Filter Operator [FIL_318] (rows=852 width=1910) predicate:((s_store_name = 'ese') and s_store_sk is not null) TableScan [TS_165] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 52 [SIMPLE_EDGE] SHUFFLE [RS_174] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_349] (rows=696954748 width=88) + Merge Join Operator [MERGEJOIN_342] (rows=696954748 width=88) Conds:RS_171._col1=RS_172._col0(Inner),Output:["_col2"] <-Map 56 [SIMPLE_EDGE] SHUFFLE [RS_172] PartitionCols:_col0 Select Operator [SEL_164] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_324] (rows=3600 width=107) + Filter Operator [FIL_317] (rows=3600 width=107) predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) TableScan [TS_162] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 51 [SIMPLE_EDGE] SHUFFLE [RS_171] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_348] (rows=633595212 width=88) + Merge Join Operator [MERGEJOIN_341] (rows=633595212 width=88) Conds:RS_168._col0=RS_169._col0(Inner),Output:["_col1","_col2"] <-Map 50 [SIMPLE_EDGE] SHUFFLE [RS_168] PartitionCols:_col0 Select Operator [SEL_158] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_322] (rows=575995635 width=88) + Filter Operator [FIL_315] (rows=575995635 width=88) predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) TableScan [TS_156] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] @@ -618,54 +618,54 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_161] (rows=14400 width=471) Output:["_col0"] - Filter Operator [FIL_323] (rows=14400 width=471) + Filter Operator [FIL_316] (rows=14400 width=471) predicate:((t_hour = 9) and (t_minute >= 30) and t_time_sk is not null) TableScan [TS_159] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] <-Reducer 62 [SIMPLE_EDGE] - SHUFFLE [RS_222] + SHUFFLE [RS_215] Group By Operator [GBY_206] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 61 [SIMPLE_EDGE] SHUFFLE [RS_205] Group By Operator [GBY_204] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_353] (rows=766650239 width=88) + Merge Join Operator [MERGEJOIN_346] (rows=766650239 width=88) Conds:RS_200._col2=RS_201._col0(Inner) <-Map 65 [SIMPLE_EDGE] SHUFFLE [RS_201] PartitionCols:_col0 Select Operator [SEL_193] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_329] (rows=852 width=1910) + Filter Operator [FIL_322] (rows=852 width=1910) predicate:((s_store_name = 'ese') and s_store_sk is not null) TableScan [TS_191] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] <-Reducer 60 [SIMPLE_EDGE] SHUFFLE [RS_200] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_352] (rows=696954748 width=88) + Merge Join Operator [MERGEJOIN_345] (rows=696954748 width=88) Conds:RS_197._col1=RS_198._col0(Inner),Output:["_col2"] <-Map 64 [SIMPLE_EDGE] SHUFFLE [RS_198] PartitionCols:_col0 Select Operator [SEL_190] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_328] (rows=3600 width=107) + Filter Operator [FIL_321] (rows=3600 width=107) predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) TableScan [TS_188] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] <-Reducer 59 [SIMPLE_EDGE] SHUFFLE [RS_197] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_351] (rows=633595212 width=88) + Merge Join Operator [MERGEJOIN_344] (rows=633595212 width=88) Conds:RS_194._col0=RS_195._col0(Inner),Output:["_col1","_col2"] <-Map 58 [SIMPLE_EDGE] SHUFFLE [RS_194] PartitionCols:_col0 Select Operator [SEL_184] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_326] (rows=575995635 width=88) + Filter Operator [FIL_319] (rows=575995635 width=88) predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) TableScan [TS_182] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] @@ -674,7 +674,7 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_187] (rows=14400 width=471) Output:["_col0"] - Filter Operator [FIL_327] (rows=14400 width=471) + Filter Operator [FIL_320] (rows=14400 width=471) predicate:((t_hour = 9) and (t_minute < 30) and t_time_sk is not null) TableScan [TS_185] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour","t_minute"] diff --git ql/src/test/results/clientpositive/perf/query90.q.out ql/src/test/results/clientpositive/perf/query90.q.out index eea2a06..7108931 100644 --- ql/src/test/results/clientpositive/perf/query90.q.out +++ ql/src/test/results/clientpositive/perf/query90.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[93][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[92][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 6' is a cross product PREHOOK: query: explain select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 6 and 6+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 14 and 14+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100 PREHOOK: type: QUERY POSTHOOK: query: explain select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 6 and 6+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 14 and 14+1 and household_demographics.hd_dep_count = 8 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio limit 100 @@ -22,61 +22,61 @@ Stage-0 limit:100 Stage-1 Reducer 7 - File Output Operator [FS_60] - Limit [LIM_59] (rows=1 width=8) + File Output Operator [FS_59] + Limit [LIM_58] (rows=1 width=8) Number of rows:100 - Select Operator [SEL_58] (rows=1 width=8) + Select Operator [SEL_57] (rows=1 width=8) Output:["_col0"] <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_57] - Select Operator [SEL_56] (rows=1 width=8) + SHUFFLE [RS_56] + Select Operator [SEL_55] (rows=1 width=8) Output:["_col0"] - Merge Join Operator [MERGEJOIN_93] (rows=1 width=8) + Merge Join Operator [MERGEJOIN_92] (rows=1 width=8) Conds:(Inner),Output:["_col0","_col1"] <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_54] + SHUFFLE [RS_53] Group By Operator [GBY_50] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 14 [SIMPLE_EDGE] SHUFFLE [RS_49] Group By Operator [GBY_48] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_92] (rows=191667562 width=135) + Merge Join Operator [MERGEJOIN_91] (rows=191667562 width=135) Conds:RS_44._col1=RS_45._col0(Inner) <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_45] PartitionCols:_col0 Select Operator [SEL_37] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_86] (rows=3600 width=107) + Filter Operator [FIL_85] (rows=3600 width=107) predicate:((hd_dep_count = 8) and hd_demo_sk is not null) TableScan [TS_35] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] <-Reducer 13 [SIMPLE_EDGE] SHUFFLE [RS_44] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_91] (rows=174243235 width=135) + Merge Join Operator [MERGEJOIN_90] (rows=174243235 width=135) Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1"] <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_42] PartitionCols:_col0 Select Operator [SEL_34] (rows=43200 width=471) Output:["_col0"] - Filter Operator [FIL_85] (rows=43200 width=471) + Filter Operator [FIL_84] (rows=43200 width=471) predicate:(t_hour BETWEEN 14 AND 15 and t_time_sk is not null) TableScan [TS_32] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour"] <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_41] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_90] (rows=158402938 width=135) + Merge Join Operator [MERGEJOIN_89] (rows=158402938 width=135) Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col0","_col1"] <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:_col2 Select Operator [SEL_28] (rows=144002668 width=135) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_83] (rows=144002668 width=135) + Filter Operator [FIL_82] (rows=144002668 width=135) predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null) TableScan [TS_26] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"] @@ -85,54 +85,54 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_31] (rows=2301 width=585) Output:["_col0"] - Filter Operator [FIL_84] (rows=2301 width=585) + Filter Operator [FIL_83] (rows=2301 width=585) predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) TableScan [TS_29] (rows=4602 width=585) default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk","wp_char_count"] <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_53] + SHUFFLE [RS_52] Group By Operator [GBY_24] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_23] Group By Operator [GBY_22] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_89] (rows=191667562 width=135) + Merge Join Operator [MERGEJOIN_88] (rows=191667562 width=135) Conds:RS_18._col1=RS_19._col0(Inner) <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 Select Operator [SEL_11] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_82] (rows=3600 width=107) + Filter Operator [FIL_81] (rows=3600 width=107) predicate:((hd_dep_count = 8) and hd_demo_sk is not null) TableScan [TS_9] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_88] (rows=174243235 width=135) + Merge Join Operator [MERGEJOIN_87] (rows=174243235 width=135) Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1"] <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 Select Operator [SEL_8] (rows=43200 width=471) Output:["_col0"] - Filter Operator [FIL_81] (rows=43200 width=471) + Filter Operator [FIL_80] (rows=43200 width=471) predicate:(t_hour BETWEEN 6 AND 7 and t_time_sk is not null) TableScan [TS_6] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour"] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_87] (rows=158402938 width=135) + Merge Join Operator [MERGEJOIN_86] (rows=158402938 width=135) Conds:RS_12._col2=RS_13._col0(Inner),Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col2 Select Operator [SEL_2] (rows=144002668 width=135) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_79] (rows=144002668 width=135) + Filter Operator [FIL_78] (rows=144002668 width=135) predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null) TableScan [TS_0] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"] @@ -141,7 +141,7 @@ Stage-0 PartitionCols:_col0 Select Operator [SEL_5] (rows=2301 width=585) Output:["_col0"] - Filter Operator [FIL_80] (rows=2301 width=585) + Filter Operator [FIL_79] (rows=2301 width=585) predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) TableScan [TS_3] (rows=4602 width=585) default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk","wp_char_count"] diff --git ql/src/test/results/clientpositive/pointlookup2.q.out ql/src/test/results/clientpositive/pointlookup2.q.out index d0ad68a..5facf9a 100644 --- ql/src/test/results/clientpositive/pointlookup2.q.out +++ ql/src/test/results/clientpositive/pointlookup2.q.out @@ -379,23 +379,27 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '2000-04-08' (type: string), _col3 (type: int), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4 + columns.types int,string,string,int,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Map Reduce @@ -408,7 +412,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string) + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -419,8 +423,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string + columns _col0,_col1,_col2,_col3,_col4 + columns.types int,string,string,int,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -428,8 +432,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string + columns _col0,_col1,_col2,_col3,_col4 + columns.types int,string,string,int,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -438,7 +442,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), '2000-04-08' (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -639,23 +643,27 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '2000-04-08' (type: string), _col3 (type: int), _col4 (type: string), '2000-04-09' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int,string,string,int,string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Map Reduce @@ -668,7 +676,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string) + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -679,8 +687,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int,string,string,int,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -688,8 +696,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4 - columns.types int,string,int,string + columns _col0,_col1,_col2,_col3,_col4,_col5 + columns.types int,string,string,int,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -698,7 +706,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), '2000-04-09' (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -730,7 +738,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain extended select * from pcr_t1 t1 join pcr_t2 t2 @@ -1032,7 +1040,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain extended select * from pcr_t1 t1 join pcr_t2 t2 diff --git ql/src/test/results/clientpositive/pointlookup3.q.out ql/src/test/results/clientpositive/pointlookup3.q.out index 39804cf..def1bc8 100644 --- ql/src/test/results/clientpositive/pointlookup3.q.out +++ ql/src/test/results/clientpositive/pointlookup3.q.out @@ -305,7 +305,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), '2001-04-08' (type: string) + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -461,23 +461,27 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3, _col4, _col5, _col6 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '2000-04-08' (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: string), '2001-04-08' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col3,_col4,_col5,_col6 - columns.types int,string,string,int,string,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7 + columns.types int,string,string,string,int,string,string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Map Reduce @@ -490,7 +494,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: string) + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: string), _col7 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -501,8 +505,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4,_col5,_col6 - columns.types int,string,string,int,string,string + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7 + columns.types int,string,string,string,int,string,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -510,8 +514,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4,_col5,_col6 - columns.types int,string,string,int,string,string + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7 + columns.types int,string,string,string,int,string,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -520,7 +524,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: string), '2001-04-08' (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -723,23 +727,27 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3, _col4, _col5, _col7 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '2000-04-08' (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: string), '2000-04-09' (type: string), _col7 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col3,_col4,_col5,_col7 - columns.types int,string,string,int,string,string - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7 + columns.types int,string,string,string,int,string,string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-2 Map Reduce @@ -752,7 +760,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col7 (type: string) + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: string), _col7 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -763,8 +771,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4,_col5,_col7 - columns.types int,string,string,int,string,string + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7 + columns.types int,string,string,string,int,string,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -772,8 +780,8 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col1,_col3,_col4,_col5,_col7 - columns.types int,string,string,int,string,string + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7 + columns.types int,string,string,string,int,string,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -782,7 +790,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), '2000-04-09' (type: string), VALUE._col6 (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: string), VALUE._col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -814,7 +822,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain extended select * from pcr_t1 t1 join pcr_t1 t2 diff --git ql/src/test/results/clientpositive/ppd_constant_expr.q.out ql/src/test/results/clientpositive/ppd_constant_expr.q.out index 17e2bab..21ea071 100644 --- ql/src/test/results/clientpositive/ppd_constant_expr.q.out +++ ql/src/test/results/clientpositive/ppd_constant_expr.q.out @@ -106,9 +106,9 @@ INSERT OVERWRITE TABLE ppd_constant_expr SELECT 4 + NULL, src1.key - NULL, NULL POSTHOOK: type: QUERY POSTHOOK: Input: default@src1 POSTHOOK: Output: default@ppd_constant_expr -POSTHOOK: Lineage: ppd_constant_expr.c1 EXPRESSION [] +POSTHOOK: Lineage: ppd_constant_expr.c1 SIMPLE [] POSTHOOK: Lineage: ppd_constant_expr.c2 EXPRESSION [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: ppd_constant_expr.c3 EXPRESSION [] +POSTHOOK: Lineage: ppd_constant_expr.c3 SIMPLE [] PREHOOK: query: SELECT ppd_constant_expr.* FROM ppd_constant_expr PREHOOK: type: QUERY PREHOOK: Input: default@ppd_constant_expr @@ -242,9 +242,9 @@ INSERT OVERWRITE TABLE ppd_constant_expr SELECT 4 + NULL, src1.key - NULL, NULL POSTHOOK: type: QUERY POSTHOOK: Input: default@src1 POSTHOOK: Output: default@ppd_constant_expr -POSTHOOK: Lineage: ppd_constant_expr.c1 EXPRESSION [] +POSTHOOK: Lineage: ppd_constant_expr.c1 SIMPLE [] POSTHOOK: Lineage: ppd_constant_expr.c2 EXPRESSION [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: ppd_constant_expr.c3 EXPRESSION [] +POSTHOOK: Lineage: ppd_constant_expr.c3 SIMPLE [] PREHOOK: query: SELECT ppd_constant_expr.* FROM ppd_constant_expr PREHOOK: type: QUERY PREHOOK: Input: default@ppd_constant_expr diff --git ql/src/test/results/clientpositive/ppd_join5.q.out ql/src/test/results/clientpositive/ppd_join5.q.out index f464c17..44607a0 100644 --- ql/src/test/results/clientpositive/ppd_join5.q.out +++ ql/src/test/results/clientpositive/ppd_join5.q.out @@ -32,7 +32,7 @@ POSTHOOK: Lineage: t1.id1 SIMPLE [] POSTHOOK: Lineage: t1.id2 SIMPLE [] POSTHOOK: Lineage: t2.d SIMPLE [] POSTHOOK: Lineage: t2.id SIMPLE [] -Warning: Shuffle Join JOIN[15][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select a.*,b.d d1,c.d d2 from t1 a join t2 b on (a.id1 = b.id) @@ -148,7 +148,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[15][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from ( select a.*,b.d d1,c.d d2 from @@ -271,7 +271,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[15][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: select * from ( select a.*,b.d d1,c.d d2 from t1 a join t2 b on (a.id1 = b.id) diff --git ql/src/test/results/clientpositive/ppd_outer_join4.q.out ql/src/test/results/clientpositive/ppd_outer_join4.q.out index ba5d187..ea0772a 100644 --- ql/src/test/results/clientpositive/ppd_outer_join4.q.out +++ ql/src/test/results/clientpositive/ppd_outer_join4.q.out @@ -38,33 +38,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((sqrt(key) <> 13.0) and (key > '10') and (key < '20') and (key > '15') and (key < '25')) (type: boolean) + predicate: ((key > '10') and (key < '20') and (key > '15') and (key < '25') and (sqrt(key) <> 13.0)) (type: boolean) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '10') and (key < '20') and (key > '15') and (key < '25') and (sqrt(key) <> 13.0)) (type: boolean) + predicate: ((sqrt(key) <> 13.0) and (key > '10') and (key < '20') and (key > '15') and (key < '25')) (type: boolean) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -85,7 +85,7 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 - Inner Join 1 to 2 + Inner Join 0 to 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) @@ -93,7 +93,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col0 (type: string) + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -402,33 +402,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((sqrt(key) <> 13.0) and (key > '10') and (key < '20') and (key > '15') and (key < '25')) (type: boolean) + predicate: ((key > '10') and (key < '20') and (key > '15') and (key < '25') and (sqrt(key) <> 13.0)) (type: boolean) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '10') and (key < '20') and (key > '15') and (key < '25') and (sqrt(key) <> 13.0)) (type: boolean) + predicate: ((sqrt(key) <> 13.0) and (key > '10') and (key < '20') and (key > '15') and (key < '25')) (type: boolean) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -449,7 +449,7 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 - Inner Join 1 to 2 + Inner Join 0 to 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) @@ -457,7 +457,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col0 (type: string) + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/ppd_outer_join5.q.out ql/src/test/results/clientpositive/ppd_outer_join5.q.out index 65ca9d1..643a589 100644 --- ql/src/test/results/clientpositive/ppd_outer_join5.q.out +++ ql/src/test/results/clientpositive/ppd_outer_join5.q.out @@ -30,7 +30,7 @@ POSTHOOK: query: create table t4 (id int, key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@t4 -Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from t1 full outer join t2 on t1.id=t2.id join t3 on t2.id=t3.id where t3.id=20 PREHOOK: type: QUERY POSTHOOK: query: explain select * from t1 full outer join t2 on t1.id=t2.id join t3 on t2.id=t3.id where t3.id=20 @@ -118,7 +118,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from t1 join t2 on (t1.id=t2.id) left outer join t3 on (t2.id=t3.id) where t2.id=20 PREHOOK: type: QUERY POSTHOOK: query: explain select * from t1 join t2 on (t1.id=t2.id) left outer join t3 on (t2.id=t3.id) where t2.id=20 @@ -202,7 +202,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from t1 join t2 on (t1.id=t2.id) left outer join t3 on (t1.id=t3.id) where t2.id=20 PREHOOK: type: QUERY POSTHOOK: query: explain select * from t1 join t2 on (t1.id=t2.id) left outer join t3 on (t1.id=t3.id) where t2.id=20 diff --git ql/src/test/results/clientpositive/ppd_repeated_alias.q.out ql/src/test/results/clientpositive/ppd_repeated_alias.q.out index 3dbd258..a50e10c 100644 --- ql/src/test/results/clientpositive/ppd_repeated_alias.q.out +++ ql/src/test/results/clientpositive/ppd_repeated_alias.q.out @@ -263,7 +263,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[15][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: -- Q4: here, the filter c.bar should be created under the first join but above the second explain select c.foo, d.bar from (select c.foo, b.bar, c.blah from pokes c left outer join pokes b on c.foo=b.foo) c left outer join pokes d where d.foo=1 and c.bar=2 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/ppd_udf_case.q.out ql/src/test/results/clientpositive/ppd_udf_case.q.out index 1c1c2a4..56941cd 100644 --- ql/src/test/results/clientpositive/ppd_udf_case.q.out +++ ql/src/test/results/clientpositive/ppd_udf_case.q.out @@ -1,3 +1,4 @@ +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM srcpart a JOIN srcpart b @@ -37,32 +38,28 @@ STAGE PLANS: alias: a Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((ds = '2008-04-08') and CASE WHEN ((key = '27')) THEN (true) WHEN ((key = '38')) THEN (false) ELSE (null) END and key is not null) (type: boolean) + predicate: ((ds = '2008-04-08') and (key = '27')) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col3 + expressions: value (type: string), hr (type: string) + outputColumnNames: _col1, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + sort order: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col3 (type: string) TableScan alias: a Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((ds = '2008-04-08') and CASE WHEN ((key = '27')) THEN (true) WHEN ((key = '38')) THEN (false) ELSE (null) END and key is not null) (type: boolean) + predicate: ((ds = '2008-04-08') and (key = '27')) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col3 + expressions: value (type: string), hr (type: string) + outputColumnNames: _col1, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + sort order: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col3 (type: string) Reduce Operator Tree: @@ -70,9 +67,9 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3, _col4, _col5, _col7 + 0 + 1 + outputColumnNames: _col1, _col3, _col5, _col7 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -86,12 +83,12 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), '2008-04-08' (type: string), _col7 (type: string) + key expressions: '27' (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string), '27' (type: string), _col5 (type: string), '2008-04-08' (type: string), _col7 (type: string) sort order: ++++++++ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey7 (type: string) + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey5 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -108,6 +105,7 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM srcpart a JOIN srcpart b ON a.key = b.key @@ -144,6 +142,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 27 val_27 2008-04-08 11 27 val_27 2008-04-08 12 27 val_27 2008-04-08 12 27 val_27 2008-04-08 11 27 val_27 2008-04-08 12 27 val_27 2008-04-08 12 +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT * FROM srcpart a JOIN srcpart b @@ -183,32 +182,28 @@ STAGE PLANS: alias: a Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (CASE WHEN ((key = '27')) THEN (true) WHEN ((key = '38')) THEN (false) ELSE (null) END and key is not null) (type: boolean) + predicate: (key = '27') (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col3 + expressions: value (type: string), hr (type: string) + outputColumnNames: _col1, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + sort order: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col3 (type: string) TableScan alias: a Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (CASE WHEN ((key = '27')) THEN (true) WHEN ((key = '38')) THEN (false) ELSE (null) END and key is not null) (type: boolean) + predicate: (key = '27') (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col3 + expressions: value (type: string), hr (type: string) + outputColumnNames: _col1, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + sort order: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col3 (type: string) Reduce Operator Tree: @@ -216,9 +211,9 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3, _col4, _col5, _col7 + 0 + 1 + outputColumnNames: _col1, _col3, _col5, _col7 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -232,12 +227,12 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), '2008-04-08' (type: string), _col7 (type: string) + key expressions: '27' (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string), '27' (type: string), _col5 (type: string), '2008-04-08' (type: string), _col7 (type: string) sort order: ++++++++ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey7 (type: string) + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey5 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey7 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -254,6 +249,7 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT * FROM srcpart a JOIN srcpart b ON a.key = b.key diff --git ql/src/test/results/clientpositive/ppd_union_view.q.out ql/src/test/results/clientpositive/ppd_union_view.q.out index 435b6f9..3081e28 100644 --- ql/src/test/results/clientpositive/ppd_union_view.q.out +++ ql/src/test/results/clientpositive/ppd_union_view.q.out @@ -156,14 +156,14 @@ STAGE PLANS: predicate: keymap is not null (type: boolean) Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: keymap (type: string), value (type: string), ds (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: keymap (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string) @@ -177,14 +177,14 @@ STAGE PLANS: predicate: keymap is not null (type: boolean) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), keymap (type: string), ds (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: key (type: string), keymap (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string), _col2 (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col2 (type: string) + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col0 (type: string) @@ -291,8 +291,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string), _col2 (type: string) - 1 _col1 (type: string), _col2 (type: string) + 0 _col0 (type: string) + 1 _col1 (type: string) outputColumnNames: _col1, _col3 Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -488,10 +488,10 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), '2011-10-15' (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), '2011-10-15' (type: string) + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE tag: 0 value expressions: _col1 (type: string) @@ -509,10 +509,10 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string), '2011-10-15' (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col1 (type: string), '2011-10-15' (type: string) + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE tag: 1 value expressions: _col0 (type: string) @@ -523,8 +523,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string), _col2 (type: string) - 1 _col1 (type: string), _col2 (type: string) + 0 _col0 (type: string) + 1 _col1 (type: string) outputColumnNames: _col1, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator diff --git ql/src/test/results/clientpositive/quotedid_basic.q.out ql/src/test/results/clientpositive/quotedid_basic.q.out index 76bd883..4f79a46 100644 --- ql/src/test/results/clientpositive/quotedid_basic.q.out +++ ql/src/test/results/clientpositive/quotedid_basic.q.out @@ -102,23 +102,23 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: x+1 (type: string), y&y (type: string) - outputColumnNames: _col0, _col1 + outputColumnNames: x+1, y&y Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string), '1' (type: string) + keys: x+1 (type: string), y&y (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), '1' (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), '1' (type: string) + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string), '1' (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), '1' (type: string) @@ -161,34 +161,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: x+1 (type: string), y&y (type: string) - outputColumnNames: _col0, _col1 + outputColumnNames: x+1, y&y Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string), '1' (type: string) + keys: x+1 (type: string), y&y (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), '1' (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), '1' (type: string) + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string), '1' (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -202,20 +198,20 @@ STAGE PLANS: value expressions: _col0 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE PTF Operator Function definitions: Input definition input alias: ptf_0 - output shape: _col0: string, _col1: string, _col2: string + output shape: _col0: string, _col1: string type: WINDOWING Windowing table definition input alias: ptf_1 name: windowingtablefunction order by: _col1 ASC NULLS FIRST - partition by: _col2 + partition by: '1' raw input shape: window functions: window function definition @@ -227,7 +223,7 @@ STAGE PLANS: isPivotResult: true Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), rank_window_0 (type: int) + expressions: _col0 (type: string), _col1 (type: string), '1' (type: string), rank_window_0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator @@ -269,34 +265,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: x+1 (type: string), y&y (type: string) - outputColumnNames: _col0, _col1 + outputColumnNames: x+1, y&y Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string), '1' (type: string) + keys: x+1 (type: string), y&y (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), '1' (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), '1' (type: string) + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string), '1' (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -310,20 +302,20 @@ STAGE PLANS: value expressions: _col0 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE PTF Operator Function definitions: Input definition input alias: ptf_0 - output shape: _col0: string, _col1: string, _col2: string + output shape: _col0: string, _col1: string type: WINDOWING Windowing table definition input alias: ptf_1 name: windowingtablefunction order by: _col1 ASC NULLS FIRST - partition by: _col2 + partition by: '1' raw input shape: window functions: window function definition @@ -335,7 +327,7 @@ STAGE PLANS: isPivotResult: true Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), rank_window_0 (type: int) + expressions: _col0 (type: string), _col1 (type: string), '1' (type: string), rank_window_0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/quotedid_partition.q.out ql/src/test/results/clientpositive/quotedid_partition.q.out index 66cff2a..fe4c5db 100644 --- ql/src/test/results/clientpositive/quotedid_partition.q.out +++ ql/src/test/results/clientpositive/quotedid_partition.q.out @@ -47,26 +47,26 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: y&y (type: string) - outputColumnNames: _col1 + outputColumnNames: y&y Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: '10' (type: string), _col1 (type: string), 'a' (type: string) + keys: y&y (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '10' (type: string), _col1 (type: string), 'a' (type: string) - sort order: +++ - Map-reduce partition columns: '10' (type: string), _col1 (type: string), 'a' (type: string) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator - keys: '10' (type: string), KEY._col1 (type: string), 'a' (type: string) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '10' (type: string), _col1 (type: string), 'a' (type: string) + expressions: '10' (type: string), _col0 (type: string), 'a' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/rand_partitionpruner3.q.out ql/src/test/results/clientpositive/rand_partitionpruner3.q.out index a2a06b2..eabf9d9 100644 --- ql/src/test/results/clientpositive/rand_partitionpruner3.q.out +++ ql/src/test/results/clientpositive/rand_partitionpruner3.q.out @@ -65,12 +65,12 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((rand(1) < 0.1) and ((UDFToDouble(key) <= 50.0) and (UDFToDouble(key) >= 10.0))) (type: boolean) - Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE + predicate: ((rand(1) < 0.1) and (not ((UDFToDouble(key) > 50.0) or (UDFToDouble(key) < 10.0)))) (type: boolean) + Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: select a.* from srcpart a where rand(1) < 0.1 and a.ds = '2008-04-08' and not(key > 50 or key < 10) and a.hr like '%2' @@ -153,12 +153,12 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: ((UDFToDouble(key) <= 50.0) and (UDFToDouble(key) >= 10.0)) (type: boolean) - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + predicate: (not ((UDFToDouble(key) > 50.0) or (UDFToDouble(key) < 10.0))) (type: boolean) + Statistics: Num rows: 168 Data size: 1784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 168 Data size: 1784 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: select a.* from srcpart a where a.ds = '2008-04-08' and not(key > 50 or key < 10) and a.hr like '%2' diff --git ql/src/test/results/clientpositive/recursive_dir.q.out ql/src/test/results/clientpositive/recursive_dir.q.out index 599b255..8789a2d 100644 --- ql/src/test/results/clientpositive/recursive_dir.q.out +++ ql/src/test/results/clientpositive/recursive_dir.q.out @@ -32,7 +32,7 @@ SELECT key+11 FROM src WHERE key=484 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@fact_tz@ds=1/hr=1 -POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=1).x EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: fact_tz PARTITION(ds=1,hr=1).x EXPRESSION [] PREHOOK: query: ALTER TABLE fact_daily SET TBLPROPERTIES('EXTERNAL'='TRUE') PREHOOK: type: ALTERTABLE_PROPERTIES PREHOOK: Input: default@fact_daily diff --git ql/src/test/results/clientpositive/semijoin4.q.out ql/src/test/results/clientpositive/semijoin4.q.out index 015dad1..e557be1 100644 --- ql/src/test/results/clientpositive/semijoin4.q.out +++ ql/src/test/results/clientpositive/semijoin4.q.out @@ -69,32 +69,32 @@ STAGE PLANS: alias: t1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((UDFToInteger(tinyint_col_46) = -92) and decimal1309_col_65 is not null and bigint_col_13 is not null and UDFToInteger(tinyint_col_46) is not null) (type: boolean) + predicate: ((UDFToInteger(tinyint_col_46) = -92) and decimal1309_col_65 is not null and bigint_col_13 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: bigint_col_13 (type: bigint), smallint_col_24 (type: smallint), double_col_60 (type: double), decimal1309_col_65 (type: decimal(13,9)) - outputColumnNames: _col0, _col1, _col3, _col4 + expressions: bigint_col_13 (type: bigint), smallint_col_24 (type: smallint), tinyint_col_46 (type: tinyint), double_col_60 (type: double), decimal1309_col_65 (type: decimal(13,9)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: bigint), _col4 (type: decimal(27,9)), -92 (type: tinyint) + key expressions: _col0 (type: bigint), _col4 (type: decimal(27,9)), _col2 (type: tinyint) sort order: +++ - Map-reduce partition columns: _col0 (type: bigint), _col4 (type: decimal(27,9)), -92 (type: tinyint) + Map-reduce partition columns: _col0 (type: bigint), _col4 (type: decimal(27,9)), _col2 (type: tinyint) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col1 (type: smallint), _col3 (type: double) TableScan alias: t2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((UDFToInteger(tinyint_col_21) = -92) and tinyint_col_18 is not null and decimal2709_col_9 is not null and UDFToInteger(tinyint_col_21) is not null) (type: boolean) + predicate: ((UDFToInteger(tinyint_col_21) = -92) and tinyint_col_18 is not null and decimal2709_col_9 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: decimal2709_col_9 (type: decimal(27,9)), tinyint_col_18 (type: tinyint) - outputColumnNames: _col0, _col1 + expressions: decimal2709_col_9 (type: decimal(27,9)), tinyint_col_18 (type: tinyint), tinyint_col_21 (type: tinyint) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: UDFToLong(_col1) (type: bigint), _col0 (type: decimal(27,9)), -92 (type: tinyint) + key expressions: UDFToLong(_col1) (type: bigint), _col0 (type: decimal(27,9)), _col2 (type: tinyint) sort order: +++ - Map-reduce partition columns: UDFToLong(_col1) (type: bigint), _col0 (type: decimal(27,9)), -92 (type: tinyint) + Map-reduce partition columns: UDFToLong(_col1) (type: bigint), _col0 (type: decimal(27,9)), _col2 (type: tinyint) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Join Operator @@ -103,11 +103,11 @@ STAGE PLANS: keys: 0 _col0 (type: bigint), _col4 (type: decimal(27,9)), _col2 (type: tinyint) 1 UDFToLong(_col1) (type: bigint), _col0 (type: decimal(27,9)), _col2 (type: tinyint) - outputColumnNames: _col1, _col3 + outputColumnNames: _col1, _col3, _col7 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: smallint), _col3 (type: double) - outputColumnNames: _col0, _col1 + expressions: _col1 (type: smallint), _col3 (type: double), _col7 (type: tinyint), UDFToInteger(_col7) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false @@ -121,16 +121,16 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: -92 (type: int) + key expressions: _col3 (type: int) sort order: + - Map-reduce partition columns: -92 (type: int) + Map-reduce partition columns: _col3 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: smallint), _col1 (type: double) + value expressions: _col0 (type: smallint), _col1 (type: double), _col2 (type: tinyint) TableScan Reduce Output Operator - key expressions: -92 (type: int) + key expressions: _col0 (type: int) sort order: + - Map-reduce partition columns: -92 (type: int) + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Join Operator @@ -139,7 +139,7 @@ STAGE PLANS: keys: 0 _col3 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false @@ -153,14 +153,14 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: (-92 + _col0) (type: smallint), floor(_col1) (type: bigint) + key expressions: (UDFToShort(_col2) + _col0) (type: smallint), floor(_col1) (type: bigint) sort order: +- - Map-reduce partition columns: (-92 + _col0) (type: smallint) + Map-reduce partition columns: (UDFToShort(_col2) + _col0) (type: smallint) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: smallint), _col1 (type: double) + value expressions: _col0 (type: smallint), _col1 (type: double), _col2 (type: tinyint) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: double), -92 (type: tinyint) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: double), VALUE._col2 (type: tinyint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE PTF Operator @@ -237,17 +237,21 @@ STAGE PLANS: 0 _col0 (type: decimal(19,11)), _col1 (type: timestamp) 1 _col0 (type: decimal(19,11)), _col1 (type: timestamp) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - keys: -92 (type: int) - mode: hash + Select Operator + expressions: -92 (type: int) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/semijoin5.q.out ql/src/test/results/clientpositive/semijoin5.q.out index 70d705a..6ecc693 100644 --- ql/src/test/results/clientpositive/semijoin5.q.out +++ ql/src/test/results/clientpositive/semijoin5.q.out @@ -78,18 +78,18 @@ STAGE PLANS: alias: t2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((UDFToInteger(smallint_col_19) = -92) and tinyint_col_20 is not null and decimal2709_col_9 is not null and tinyint_col_15 is not null and UDFToInteger(smallint_col_19) is not null) (type: boolean) + predicate: ((UDFToInteger(smallint_col_19) = -92) and tinyint_col_20 is not null and decimal2709_col_9 is not null and tinyint_col_15 is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: decimal2709_col_9 (type: decimal(27,9)), int_col_10 (type: int), tinyint_col_15 (type: tinyint), tinyint_col_20 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col4 + expressions: decimal2709_col_9 (type: decimal(27,9)), int_col_10 (type: int), tinyint_col_15 (type: tinyint), smallint_col_19 (type: smallint), tinyint_col_20 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator key expressions: UDFToLong(_col2) (type: bigint), _col0 (type: decimal(34,16)), _col4 (type: tinyint) sort order: +++ Map-reduce partition columns: UDFToLong(_col2) (type: bigint), _col0 (type: decimal(34,16)), _col4 (type: tinyint) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: int) + value expressions: _col1 (type: int), _col3 (type: smallint) Reduce Operator Tree: Join Operator condition map: @@ -97,11 +97,11 @@ STAGE PLANS: keys: 0 _col1 (type: bigint), _col4 (type: decimal(34,16)), _col0 (type: tinyint) 1 UDFToLong(_col2) (type: bigint), _col0 (type: decimal(34,16)), _col4 (type: tinyint) - outputColumnNames: _col2, _col3, _col5, _col7 + outputColumnNames: _col2, _col3, _col5, _col7, _col9 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col2 (type: timestamp), _col3 (type: double), _col5 (type: smallint), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: _col2 (type: timestamp), _col3 (type: double), _col5 (type: smallint), _col7 (type: int), UDFToInteger(_col9) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false @@ -115,16 +115,16 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col0 (type: timestamp), -92 (type: int) + key expressions: _col0 (type: timestamp), _col4 (type: int) sort order: ++ - Map-reduce partition columns: _col0 (type: timestamp), -92 (type: int) + Map-reduce partition columns: _col0 (type: timestamp), _col4 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col1 (type: double), _col2 (type: smallint), _col3 (type: int) TableScan Reduce Output Operator - key expressions: _col0 (type: timestamp), -92 (type: int) + key expressions: _col0 (type: timestamp), _col1 (type: int) sort order: ++ - Map-reduce partition columns: _col0 (type: timestamp), -92 (type: int) + Map-reduce partition columns: _col0 (type: timestamp), _col1 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Join Operator @@ -280,11 +280,11 @@ STAGE PLANS: outputColumnNames: _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col2 (type: timestamp) - outputColumnNames: _col0 + expressions: _col2 (type: timestamp), -92 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - keys: _col0 (type: timestamp), -92 (type: int) + keys: _col0 (type: timestamp), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE diff --git ql/src/test/results/clientpositive/skewjoin.q.out ql/src/test/results/clientpositive/skewjoin.q.out index bd954ef..9f07a10 100644 --- ql/src/test/results/clientpositive/skewjoin.q.out +++ ql/src/test/results/clientpositive/skewjoin.q.out @@ -790,9 +790,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + UDFToDouble(1)) (type: double) + key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) sort order: ++ - Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + UDFToDouble(1)) (type: double) + Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reduce Operator Tree: @@ -802,7 +802,7 @@ STAGE PLANS: handleSkewJoin: true keys: 0 _col0 (type: string), UDFToDouble(substring(_col1, 5)) (type: double) - 1 _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + UDFToDouble(1)) (type: double) + 1 _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) outputColumnNames: _col2, _col3 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Select Operator diff --git ql/src/test/results/clientpositive/smb_mapjoin_25.q.out ql/src/test/results/clientpositive/smb_mapjoin_25.q.out index b0db59e..29c6d31 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_25.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_25.q.out @@ -46,8 +46,9 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket_3 -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -Warning: Shuffle Join JOIN[20][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-4:MAPRED' is a cross product +Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[18][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-4:MAPRED' is a cross product PREHOOK: query: explain select * from (select a.key from smb_bucket_1 a join smb_bucket_2 b on (a.key = b.key) where a.key = 5) t1 left outer join (select c.key from smb_bucket_2 c join smb_bucket_3 d on (c.key = d.key) where c.key=5) t2 on (t1.key=t2.key) where t2.key=5 PREHOOK: type: QUERY @@ -106,23 +107,19 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: 5 (type: int) - sort order: + - Map-reduce partition columns: 5 (type: int) + sort order: Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator - key expressions: 5 (type: int) - sort order: + - Map-reduce partition columns: 5 (type: int) + sort order: Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 5 (type: int) - 1 5 (type: int) + 0 + 1 Statistics: Num rows: 31 Data size: 129 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 5 (type: int), 5 (type: int) @@ -182,8 +179,11 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[37][bigTable=?] in task 'Stage-9:MAPRED' is a cross product -Warning: Map Join MAPJOIN[38][bigTable=?] in task 'Stage-10:MAPRED' is a cross product +Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[35][bigTable=?] in task 'Stage-9:MAPRED' is a cross product +Warning: Map Join MAPJOIN[36][bigTable=?] in task 'Stage-10:MAPRED' is a cross product PREHOOK: query: -- explain -- select * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key left outer join src c on a.key=c.value @@ -276,8 +276,8 @@ STAGE PLANS: TableScan HashTable Sink Operator keys: - 0 5 (type: int) - 1 5 (type: int) + 0 + 1 Stage: Stage-6 Map Reduce @@ -287,8 +287,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 5 (type: int) - 1 5 (type: int) + 0 + 1 Statistics: Num rows: 31 Data size: 129 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 5 (type: int), 5 (type: int) @@ -315,8 +315,8 @@ STAGE PLANS: TableScan HashTable Sink Operator keys: - 0 5 (type: int) - 1 5 (type: int) + 0 + 1 Stage: Stage-7 Map Reduce @@ -326,8 +326,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 5 (type: int) - 1 5 (type: int) + 0 + 1 Statistics: Num rows: 31 Data size: 129 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 5 (type: int), 5 (type: int) @@ -348,23 +348,19 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: 5 (type: int) - sort order: + - Map-reduce partition columns: 5 (type: int) + sort order: Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator - key expressions: 5 (type: int) - sort order: + - Map-reduce partition columns: 5 (type: int) + sort order: Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 5 (type: int) - 1 5 (type: int) + 0 + 1 Statistics: Num rows: 31 Data size: 129 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 5 (type: int), 5 (type: int) @@ -432,8 +428,11 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[37][bigTable=?] in task 'Stage-9:MAPRED' is a cross product -Warning: Map Join MAPJOIN[38][bigTable=?] in task 'Stage-10:MAPRED' is a cross product +Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[35][bigTable=?] in task 'Stage-9:MAPRED' is a cross product +Warning: Map Join MAPJOIN[36][bigTable=?] in task 'Stage-10:MAPRED' is a cross product PREHOOK: query: select * from (select a.key from smb_bucket_1 a join smb_bucket_2 b on (a.key = b.key) where a.key = 5) t1 left outer join (select c.key from smb_bucket_2 c join smb_bucket_3 d on (c.key = d.key) where c.key=5) t2 on (t1.key=t2.key) where t2.key=5 PREHOOK: type: QUERY PREHOOK: Input: default@smb_bucket_1 diff --git ql/src/test/results/clientpositive/spark/auto_join8.q.out ql/src/test/results/clientpositive/spark/auto_join8.q.out index a769f4c..5427a0a 100644 --- ql/src/test/results/clientpositive/spark/auto_join8.q.out +++ ql/src/test/results/clientpositive/spark/auto_join8.q.out @@ -157,7 +157,7 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.c3 EXPRESSION [] POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/auto_join_filters.q.out ql/src/test/results/clientpositive/spark/auto_join_filters.q.out index 84810d5..85b2a32 100644 --- ql/src/test/results/clientpositive/spark/auto_join_filters.q.out +++ ql/src/test/results/clientpositive/spark/auto_join_filters.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -300,7 +300,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in2.txt' into table sm POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_input2 -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/spark/auto_join_nulls.q.out ql/src/test/results/clientpositive/spark/auto_join_nulls.q.out index 15f4791..32a885b 100644 --- ql/src/test/results/clientpositive/spark/auto_join_nulls.q.out +++ ql/src/test/results/clientpositive/spark/auto_join_nulls.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -24,7 +24,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out index eff3671..e8ba2d1 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out @@ -138,7 +138,7 @@ POSTHOOK: query: load data local inpath '../../data/files/smallsrcsortbucket3out POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@bucket_medium@ds=2008-04-08 -Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY POSTHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key @@ -563,7 +563,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@bucket_big diff --git ql/src/test/results/clientpositive/spark/bucketizedhiveinputformat.q.out ql/src/test/results/clientpositive/spark/bucketizedhiveinputformat.q.out index f164f9d..518adca 100644 --- ql/src/test/results/clientpositive/spark/bucketizedhiveinputformat.q.out +++ ql/src/test/results/clientpositive/spark/bucketizedhiveinputformat.q.out @@ -22,7 +22,7 @@ POSTHOOK: query: CREATE TABLE T2(name STRING) STORED AS SEQUENCEFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@T2 -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 2' is a cross product PREHOOK: query: INSERT OVERWRITE TABLE T2 SELECT * FROM ( SELECT tmp1.name as name FROM ( SELECT name, 'MMM' AS n FROM T1) tmp1 diff --git ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out index 85387a7..de829e2 100644 --- ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out +++ ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out @@ -434,7 +434,7 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid <> 100) and (dimid = 100) is not null) (type: boolean) + predicate: (((dimid = 100) = true) and (dimid <> 100)) (type: boolean) Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) @@ -452,10 +452,10 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id <> 100) and (id = 100) is not null) (type: boolean) + predicate: (((id = 100) = true) and (id <> 100)) (type: boolean) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), (id = 100) (type: boolean) + expressions: id (type: int), true (type: boolean) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -523,7 +523,7 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((dimid) IN (100, 200) and ((dimid = 100) = true) and (dimid = 100) is not null) (type: boolean) + predicate: ((dimid) IN (100, 200) and ((dimid = 100) = true)) (type: boolean) Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) @@ -541,10 +541,10 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((id) IN (100, 200) and ((id = 100) = true) and (id = 100) is not null) (type: boolean) + predicate: ((id) IN (100, 200) and ((id = 100) = true)) (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), (id = 100) (type: boolean) + expressions: id (type: int), true (type: boolean) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -614,7 +614,7 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid = 200) and (dimid = 100) is not null) (type: boolean) + predicate: (((dimid = 100) = true) and (dimid = 200)) (type: boolean) Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string) @@ -632,19 +632,21 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id = 200) and (id = 100) is not null) (type: boolean) + predicate: (((id = 100) = true) and (id = 200)) (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator + expressions: 200 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: 200 (type: int), false (type: boolean) + keys: _col0 (type: int), _col1 (type: boolean) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 200 (type: int), false (type: boolean) + key expressions: _col0 (type: int), _col1 (type: boolean) sort order: ++ - Map-reduce partition columns: 200 (type: int), false (type: boolean) + Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: @@ -652,7 +654,7 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 keys: - 0 _col3 (type: int), true (type: boolean) + 0 200 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE @@ -701,7 +703,7 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and (dimid = 100) and (dimid = 100) is not null) (type: boolean) + predicate: (((dimid = 100) = true) and (dimid = 100)) (type: boolean) Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string) @@ -719,19 +721,21 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and (id = 100) and (id = 100) is not null) (type: boolean) + predicate: (((id = 100) = true) and (id = 100)) (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator + expressions: 100 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: 100 (type: int), true (type: boolean) + keys: _col0 (type: int), _col1 (type: boolean) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 100 (type: int), true (type: boolean) + key expressions: _col0 (type: int), _col1 (type: boolean) sort order: ++ - Map-reduce partition columns: 100 (type: int), true (type: boolean) + Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: @@ -739,7 +743,7 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 keys: - 0 _col3 (type: int), true (type: boolean) + 0 100 (type: int), true (type: boolean) 1 _col0 (type: int), _col1 (type: boolean) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE @@ -790,7 +794,7 @@ STAGE PLANS: alias: table1 Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((dimid = 100) = true) and dimid is not null and (dimid = 100) is not null) (type: boolean) + predicate: (((dimid = 100) = true) and dimid is not null) (type: boolean) Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int) @@ -808,10 +812,10 @@ STAGE PLANS: alias: table3 Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((id = 100) = true) and id is not null and (id = 100) is not null) (type: boolean) + predicate: (((id = 100) = true) and id is not null) (type: boolean) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: id (type: int), (id = 100) (type: boolean) + expressions: id (type: int), true (type: boolean) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Group By Operator diff --git ql/src/test/results/clientpositive/spark/cross_join.q.out ql/src/test/results/clientpositive/spark/cross_join.q.out index 2740c18..6b9a4be 100644 --- ql/src/test/results/clientpositive/spark/cross_join.q.out +++ ql/src/test/results/clientpositive/spark/cross_join.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: -- current explain select src.key from src join src src2 PREHOOK: type: QUERY @@ -63,7 +63,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: -- ansi cross join explain select src.key from src cross join src src2 PREHOOK: type: QUERY @@ -203,7 +203,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select src.key from src join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src join src src2 @@ -271,7 +271,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select src.key from src cross join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src cross join src src2 diff --git ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out index 65f0c22..9a965dd 100644 --- ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out +++ ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out @@ -32,7 +32,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -98,7 +98,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -202,7 +202,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -328,8 +328,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[9][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -438,7 +438,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 diff --git ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out index 26bee4e..af4f968 100644 --- ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out +++ ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out @@ -32,7 +32,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -102,7 +102,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -209,7 +209,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[23][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -343,8 +343,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -464,7 +464,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 diff --git ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out index 8163773..86c5bd7 100644 --- ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out +++ ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out @@ -908,46 +908,46 @@ STAGE PLANS: outputColumnNames: _col2, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col6 (type: string), _col5 (type: int), _col4 (type: int), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col4 + expressions: _col4 (type: int), _col5 (type: int), _col6 (type: string), _col2 (type: int) + outputColumnNames: _col4, _col5, _col6, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: avg(_col4), stddev_samp(_col4) - keys: _col0 (type: string), _col1 (type: int), _col2 (type: int), 4 (type: int) + aggregations: stddev_samp(_col2), avg(_col2) + keys: _col4 (type: int), _col5 (type: int), _col6 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), 4 (type: int) - sort order: ++++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: int), 4 (type: int) + key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col4 (type: struct), _col5 (type: struct) + value expressions: _col3 (type: struct), _col4 (type: struct) Reducer 15 Reduce Operator Tree: Group By Operator - aggregations: avg(VALUE._col0), stddev_samp(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: int), 4 (type: int) + aggregations: stddev_samp(VALUE._col0), avg(VALUE._col1) + keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), _col5 (type: double) - outputColumnNames: _col1, _col2, _col4, _col5 + expressions: _col1 (type: int), _col0 (type: int), _col3 (type: double), _col4 (type: double) + outputColumnNames: _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col5 / _col4) > 1.0)) END (type: boolean) + predicate: CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), CASE WHEN ((_col4 = 0.0)) THEN (null) ELSE ((_col5 / _col4)) END (type: double) - outputColumnNames: _col1, _col2, _col4, _col5 + expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), CASE WHEN ((_col4 = 0.0)) THEN (null) ELSE ((_col3 / _col4)) END (type: double) + outputColumnNames: _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col4 (type: double), _col5 (type: double) + value expressions: _col3 (type: double), _col4 (type: double) Reducer 2 Reduce Operator Tree: Join Operator @@ -991,46 +991,46 @@ STAGE PLANS: outputColumnNames: _col2, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col6 (type: string), _col5 (type: int), _col4 (type: int), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col4 + expressions: _col4 (type: int), _col5 (type: int), _col6 (type: string), _col2 (type: int) + outputColumnNames: _col4, _col5, _col6, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - aggregations: avg(_col4), stddev_samp(_col4) - keys: _col0 (type: string), _col1 (type: int), _col2 (type: int), 3 (type: int) + aggregations: stddev_samp(_col2), avg(_col2) + keys: _col4 (type: int), _col5 (type: int), _col6 (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), 3 (type: int) - sort order: ++++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: int), 3 (type: int) + key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col4 (type: struct), _col5 (type: struct) + value expressions: _col3 (type: struct), _col4 (type: struct) Reducer 5 Reduce Operator Tree: Group By Operator - aggregations: avg(VALUE._col0), stddev_samp(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: int), 3 (type: int) + aggregations: stddev_samp(VALUE._col0), avg(VALUE._col1) + keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), _col5 (type: double) - outputColumnNames: _col1, _col2, _col4, _col5 + expressions: _col1 (type: int), _col0 (type: int), _col3 (type: double), _col4 (type: double) + outputColumnNames: _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col5 / _col4) > 1.0)) END (type: boolean) + predicate: CASE WHEN ((_col4 = 0.0)) THEN (false) ELSE (((_col3 / _col4) > 1.0)) END (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), CASE WHEN ((_col4 = 0.0)) THEN (null) ELSE ((_col5 / _col4)) END (type: double) - outputColumnNames: _col1, _col2, _col4, _col5 + expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), CASE WHEN ((_col4 = 0.0)) THEN (null) ELSE ((_col3 / _col4)) END (type: double) + outputColumnNames: _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col2 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col4 (type: double), _col5 (type: double) + value expressions: _col3 (type: double), _col4 (type: double) Reducer 6 Reduce Operator Tree: Join Operator @@ -1039,10 +1039,10 @@ STAGE PLANS: keys: 0 _col2 (type: int), _col1 (type: int) 1 _col2 (type: int), _col1 (type: int) - outputColumnNames: _col1, _col2, _col4, _col5, _col7, _col8, _col10, _col11 + outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col9 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), _col2 (type: int), _col4 (type: double), _col5 (type: double), _col7 (type: int), _col8 (type: int), _col10 (type: double), _col11 (type: double) + expressions: _col1 (type: int), _col2 (type: int), _col3 (type: double), _col4 (type: double), _col6 (type: int), _col7 (type: int), _col8 (type: double), _col9 (type: double) outputColumnNames: _col0, _col1, _col3, _col4, _col5, _col6, _col8, _col9 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator @@ -1053,7 +1053,7 @@ STAGE PLANS: Reducer 7 Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), 3 (type: int), KEY.reducesinkkey3 (type: double), KEY.reducesinkkey4 (type: double), VALUE._col0 (type: int), VALUE._col1 (type: int), 4 (type: int), KEY.reducesinkkey6 (type: double), KEY.reducesinkkey7 (type: double) + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: double), KEY.reducesinkkey4 (type: double), VALUE._col0 (type: int), VALUE._col1 (type: int), KEY.reducesinkkey5 (type: int), KEY.reducesinkkey6 (type: double), KEY.reducesinkkey7 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out index a21ea9c..4c4ae03 100644 --- ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out +++ ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out @@ -836,16 +836,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string) - outputColumnNames: _col1 + outputColumnNames: key Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: 1 (type: int), _col1 (type: string) + aggregations: count(key) + keys: key (type: string) mode: final - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 1 (type: int), UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int) + expressions: 1 (type: int), UDFToInteger(_col0) (type: int), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -974,7 +974,7 @@ SELECT 1, key, count(1) FROM T1 GROUP BY 1, key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Output: default@outputtbl3 -POSTHOOK: Lineage: outputtbl3.cnt EXPRESSION [(t1)t1.null, ] +POSTHOOK: Lineage: outputtbl3.cnt EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl3.key1 SIMPLE [] POSTHOOK: Lineage: outputtbl3.key2 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: SELECT * FROM outputTbl3 @@ -1028,22 +1028,22 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) - null sort order: aaa - sort order: +++ - Map-reduce partition columns: _col0 (type: string), 1 (type: int), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col3 (type: bigint) + value expressions: _col2 (type: bigint) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1103,12 +1103,12 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1184,7 +1184,7 @@ SELECT key, 1, val, count(1) FROM T1 GROUP BY key, 1, val POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Output: default@outputtbl4 -POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t1)t1.null, ] +POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t1)t1.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl4.key3 SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ] @@ -3074,16 +3074,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -3212,7 +3212,7 @@ SELECT key, 1, val, count(1) FROM T2 GROUP BY key, 1, val POSTHOOK: type: QUERY POSTHOOK: Input: default@t2 POSTHOOK: Output: default@outputtbl4 -POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t2)t2.null, ] +POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key1 EXPRESSION [(t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl4.key3 SIMPLE [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] @@ -3268,16 +3268,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string), 2 (type: int) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), 2 (type: int), UDFToInteger(_col4) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), 2 (type: int), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -3406,7 +3406,7 @@ SELECT key, 1, val, 2, count(1) FROM T2 GROUP BY key, 1, val, 2 POSTHOOK: type: QUERY POSTHOOK: Input: default@t2 POSTHOOK: Output: default@outputtbl5 -POSTHOOK: Lineage: outputtbl5.cnt EXPRESSION [(t2)t2.null, ] +POSTHOOK: Lineage: outputtbl5.cnt EXPRESSION [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl5.key1 EXPRESSION [(t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl5.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl5.key3 SIMPLE [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] @@ -3459,16 +3459,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -3657,16 +3657,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - keys: _col0 (type: string), 2 (type: int), _col2 (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 2 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 2 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out index d0f5952..e575e97 100644 --- ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out +++ ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out @@ -855,16 +855,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string) - outputColumnNames: _col1 + outputColumnNames: key Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: 1 (type: int), _col1 (type: string) + aggregations: count(key) + keys: key (type: string) mode: final - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 1 (type: int), UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int) + expressions: 1 (type: int), UDFToInteger(_col0) (type: int), UDFToInteger(_col1) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -993,7 +993,7 @@ SELECT 1, key, count(1) FROM T1 GROUP BY 1, key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Output: default@outputtbl3 -POSTHOOK: Lineage: outputtbl3.cnt EXPRESSION [(t1)t1.null, ] +POSTHOOK: Lineage: outputtbl3.cnt EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl3.key1 SIMPLE [] POSTHOOK: Lineage: outputtbl3.key2 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: SELECT * FROM outputTbl3 @@ -1048,22 +1048,22 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) - null sort order: aaa - sort order: +++ + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: ++ Map-reduce partition columns: rand() (type: double) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col3 (type: bigint) + value expressions: _col2 (type: bigint) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1123,30 +1123,30 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: partials - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) - null sort order: aaa - sort order: +++ - Map-reduce partition columns: _col0 (type: string), 1 (type: int), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col3 (type: bigint) + value expressions: _col2 (type: bigint) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1222,7 +1222,7 @@ SELECT key, 1, val, count(1) FROM T1 GROUP BY key, 1, val POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Output: default@outputtbl4 -POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t1)t1.null, ] +POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t1)t1.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key1 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl4.key3 SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ] @@ -3207,16 +3207,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -3345,7 +3345,7 @@ SELECT key, 1, val, count(1) FROM T2 GROUP BY key, 1, val POSTHOOK: type: QUERY POSTHOOK: Input: default@t2 POSTHOOK: Output: default@outputtbl4 -POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t2)t2.null, ] +POSTHOOK: Lineage: outputtbl4.cnt EXPRESSION [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key1 EXPRESSION [(t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl4.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl4.key3 SIMPLE [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] @@ -3401,16 +3401,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: key, val Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string), 2 (type: int) + aggregations: count(val) + keys: key (type: string), val (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), 2 (type: int), UDFToInteger(_col4) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), 2 (type: int), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -3539,7 +3539,7 @@ SELECT key, 1, val, 2, count(1) FROM T2 GROUP BY key, 1, val, 2 POSTHOOK: type: QUERY POSTHOOK: Input: default@t2 POSTHOOK: Output: default@outputtbl5 -POSTHOOK: Lineage: outputtbl5.cnt EXPRESSION [(t2)t2.null, ] +POSTHOOK: Lineage: outputtbl5.cnt EXPRESSION [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl5.key1 EXPRESSION [(t2)t2.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: outputtbl5.key2 SIMPLE [] POSTHOOK: Lineage: outputtbl5.key3 SIMPLE [(t2)t2.FieldSchema(name:val, type:string, comment:null), ] @@ -3592,16 +3592,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - keys: _col0 (type: string), 1 (type: int), _col2 (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 1 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -3790,16 +3790,16 @@ STAGE PLANS: GatherStats: false Select Operator expressions: key (type: string), val (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - keys: _col0 (type: string), 2 (type: int), _col2 (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: final - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), 2 (type: int), _col2 (type: string), UDFToInteger(_col3) (type: int) + expressions: UDFToInteger(_col0) (type: int), 2 (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/spark/join38.q.out ql/src/test/results/clientpositive/spark/join38.q.out index 71d59e2..5177dca 100644 --- ql/src/test/results/clientpositive/spark/join38.q.out +++ ql/src/test/results/clientpositive/spark/join38.q.out @@ -15,17 +15,17 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@tmp POSTHOOK: Lineage: tmp.col0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col10 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col11 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col3 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col4 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col5 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col6 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col7 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col8 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: tmp.col9 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp.col1 SIMPLE [] +POSTHOOK: Lineage: tmp.col10 SIMPLE [] +POSTHOOK: Lineage: tmp.col11 SIMPLE [] +POSTHOOK: Lineage: tmp.col2 SIMPLE [] +POSTHOOK: Lineage: tmp.col3 SIMPLE [] +POSTHOOK: Lineage: tmp.col4 SIMPLE [] +POSTHOOK: Lineage: tmp.col5 SIMPLE [] +POSTHOOK: Lineage: tmp.col6 SIMPLE [] +POSTHOOK: Lineage: tmp.col7 SIMPLE [] +POSTHOOK: Lineage: tmp.col8 SIMPLE [] +POSTHOOK: Lineage: tmp.col9 SIMPLE [] PREHOOK: query: select * from tmp PREHOOK: type: QUERY PREHOOK: Input: default@tmp diff --git ql/src/test/results/clientpositive/spark/join8.q.out ql/src/test/results/clientpositive/spark/join8.q.out index 270053c..412a9eb 100644 --- ql/src/test/results/clientpositive/spark/join8.q.out +++ ql/src/test/results/clientpositive/spark/join8.q.out @@ -161,7 +161,7 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.c3 EXPRESSION [] POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT dest1.* FROM dest1 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out index 2c8034f..ee636a0 100644 --- ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out +++ ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select p1.p_name, p2.p_name from part p1 , part p2 PREHOOK: type: QUERY @@ -258,7 +258,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select p1.p_name, p2.p_name, p3.p_name from part p1 , part p2 , part p3 where p2.p_partkey + p1.p_partkey = p1.p_partkey and p3.p_name = p2.p_name diff --git ql/src/test/results/clientpositive/spark/join_cond_pushdown_1.q.out ql/src/test/results/clientpositive/spark/join_cond_pushdown_1.q.out index 0610d13..a9bbe15 100644 --- ql/src/test/results/clientpositive/spark/join_cond_pushdown_1.q.out +++ ql/src/test/results/clientpositive/spark/join_cond_pushdown_1.q.out @@ -192,7 +192,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 on p2.p_partkey + p1.p_partkey = p1.p_partkey and p3.p_name = p2.p_name PREHOOK: type: QUERY @@ -301,7 +301,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 on p2.p_partkey = 1 and p3.p_name = p2.p_name PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/join_cond_pushdown_3.q.out ql/src/test/results/clientpositive/spark/join_cond_pushdown_3.q.out index 0e748fb..ea5895b 100644 --- ql/src/test/results/clientpositive/spark/join_cond_pushdown_3.q.out +++ ql/src/test/results/clientpositive/spark/join_cond_pushdown_3.q.out @@ -196,7 +196,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 where p2.p_partkey + p1.p_partkey = p1.p_partkey and p3.p_name = p2.p_name @@ -307,7 +307,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from part p1 join part p2 join part p3 where p2.p_partkey = 1 and p3.p_name = p2.p_name diff --git ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual1.q.out ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual1.q.out index 45fba92..303c77f 100644 --- ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual1.q.out +++ ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual1.q.out @@ -248,7 +248,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from part p1 join part2 p2 join part3 p3 on p2_partkey + p_partkey = p1.p_partkey and p3_name = p2_name PREHOOK: type: QUERY @@ -357,7 +357,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from part p1 join part2 p2 join part3 p3 on p2_partkey = 1 and p3_name = p2_name PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual3.q.out ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual3.q.out index 9211cb6..641929c 100644 --- ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual3.q.out +++ ql/src/test/results/clientpositive/spark/join_cond_pushdown_unqual3.q.out @@ -252,7 +252,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from part p1 join part2 p2 join part3 p3 where p2_partkey + p1.p_partkey = p1.p_partkey and p3_name = p2_name @@ -363,7 +363,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from part p1 join part2 p2 join part3 p3 where p2_partkey = 1 and p3_name = p2_name diff --git ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out index 2d5d7c1..82f31e4 100644 --- ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out +++ ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out @@ -112,7 +112,7 @@ STAGE PLANS: predicate: (value = 50) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), 50 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -184,7 +184,7 @@ STAGE PLANS: predicate: (value = 60) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), 60 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -343,7 +343,7 @@ STAGE PLANS: predicate: (value = 50) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), 50 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -483,7 +483,7 @@ STAGE PLANS: predicate: (value = 60) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), 60 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -642,7 +642,7 @@ STAGE PLANS: predicate: (value = 50) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), 50 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -782,7 +782,7 @@ STAGE PLANS: predicate: (value = 60) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), 60 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1077,7 +1077,7 @@ STAGE PLANS: predicate: (value = 60) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), 60 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1149,7 +1149,7 @@ STAGE PLANS: predicate: (value = 40) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), 40 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1371,7 +1371,7 @@ STAGE PLANS: predicate: (value = 50) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), 50 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1443,7 +1443,7 @@ STAGE PLANS: predicate: (value = 60) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), 60 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1515,7 +1515,7 @@ STAGE PLANS: predicate: (value = 40) (type: boolean) Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), 40 (type: int) + expressions: key (type: int), value (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator diff --git ql/src/test/results/clientpositive/spark/join_reorder.q.out ql/src/test/results/clientpositive/spark/join_reorder.q.out index df6a36e..cc9f9a5 100644 --- ql/src/test/results/clientpositive/spark/join_reorder.q.out +++ ql/src/test/results/clientpositive/spark/join_reorder.q.out @@ -98,9 +98,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (UDFToDouble(_col0) + UDFToDouble(1)) (type: double) + key expressions: (UDFToDouble(_col0) + 1.0) (type: double) sort order: + - Map-reduce partition columns: (UDFToDouble(_col0) + UDFToDouble(1)) (type: double) + Map-reduce partition columns: (UDFToDouble(_col0) + 1.0) (type: double) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Reducer 2 @@ -110,7 +110,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 (UDFToDouble(_col0) + UDFToDouble(1)) (type: double) + 1 (UDFToDouble(_col0) + 1.0) (type: double) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/spark/join_view.q.out ql/src/test/results/clientpositive/spark/join_view.q.out index 14a28be..9d8f6d7 100644 --- ql/src/test/results/clientpositive/spark/join_view.q.out +++ ql/src/test/results/clientpositive/spark/join_view.q.out @@ -38,6 +38,7 @@ POSTHOOK: Input: default@invites POSTHOOK: Input: default@invites2 POSTHOOK: Output: database:default POSTHOOK: Output: default@v +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from v where ds='2011-09-01' PREHOOK: type: QUERY POSTHOOK: query: explain select * from v where ds='2011-09-01' @@ -50,7 +51,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -66,9 +67,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: '2011-09-01' (type: string) - sort order: + - Map-reduce partition columns: '2011-09-01' (type: string) + sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col0 (type: string) Map 3 @@ -84,9 +83,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: '2011-09-01' (type: string) - sort order: + - Map-reduce partition columns: '2011-09-01' (type: string) + sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col0 (type: int) Reducer 2 @@ -95,8 +92,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col1 (type: string) + 0 + 1 outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator diff --git ql/src/test/results/clientpositive/spark/pcr.q.out ql/src/test/results/clientpositive/spark/pcr.q.out index 21b7519..cd16787 100644 --- ql/src/test/results/clientpositive/spark/pcr.q.out +++ ql/src/test/results/clientpositive/spark/pcr.q.out @@ -1547,7 +1547,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: 14 (type: int), KEY.reducesinkkey1 (type: string) + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2445,19 +2445,23 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '2000-04-08' (type: string), _col3 (type: int), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string) - auto parallelism: false + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string) + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), '2000-04-08' (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2710,19 +2714,23 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: a - sort order: + + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '2000-04-08' (type: string), _col3 (type: int), _col4 (type: string), '2000-04-09' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string) - auto parallelism: false + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string) + auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), '2000-04-08' (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), '2000-04-09' (type: string) + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: string), VALUE._col4 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 22 Data size: 176 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -4159,7 +4167,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), '2008-04-08' (type: string), KEY.reducesinkkey2 (type: string) + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -4344,7 +4352,7 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), '11' (type: string) + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/spark/ppd_join5.q.out ql/src/test/results/clientpositive/spark/ppd_join5.q.out index 1c2b592..f08eb4d 100644 --- ql/src/test/results/clientpositive/spark/ppd_join5.q.out +++ ql/src/test/results/clientpositive/spark/ppd_join5.q.out @@ -32,7 +32,7 @@ POSTHOOK: Lineage: t1.id1 SIMPLE [] POSTHOOK: Lineage: t1.id2 SIMPLE [] POSTHOOK: Lineage: t2.d SIMPLE [] POSTHOOK: Lineage: t2.id SIMPLE [] -Warning: Shuffle Join JOIN[15][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select a.*,b.d d1,c.d d2 from t1 a join t2 b on (a.id1 = b.id) @@ -148,7 +148,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[15][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from ( select a.*,b.d d1,c.d d2 from @@ -271,7 +271,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[15][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product PREHOOK: query: select * from ( select a.*,b.d d1,c.d d2 from t1 a join t2 b on (a.id1 = b.id) diff --git ql/src/test/results/clientpositive/spark/ppd_outer_join4.q.out ql/src/test/results/clientpositive/spark/ppd_outer_join4.q.out index 312b3bd..ca6d1f8 100644 --- ql/src/test/results/clientpositive/spark/ppd_outer_join4.q.out +++ ql/src/test/results/clientpositive/spark/ppd_outer_join4.q.out @@ -43,35 +43,35 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((sqrt(key) <> 13.0) and (key > '10') and (key < '20') and (key > '15') and (key < '25')) (type: boolean) + predicate: ((key > '10') and (key < '20') and (key > '15') and (key < '25') and (sqrt(key) <> 13.0)) (type: boolean) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Map 3 Map Operator Tree: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '10') and (key < '20') and (key > '15') and (key < '25') and (sqrt(key) <> 13.0)) (type: boolean) + predicate: ((sqrt(key) <> 13.0) and (key > '10') and (key < '20') and (key > '15') and (key < '25')) (type: boolean) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) Map 4 Map Operator Tree: TableScan @@ -95,7 +95,7 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 - Inner Join 1 to 2 + Inner Join 0 to 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) @@ -103,7 +103,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col0 (type: string) + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -417,35 +417,35 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((sqrt(key) <> 13.0) and (key > '10') and (key < '20') and (key > '15') and (key < '25')) (type: boolean) + predicate: ((key > '10') and (key < '20') and (key > '15') and (key < '25') and (sqrt(key) <> 13.0)) (type: boolean) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Map 3 Map Operator Tree: TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '10') and (key < '20') and (key > '15') and (key < '25') and (sqrt(key) <> 13.0)) (type: boolean) + predicate: ((sqrt(key) <> 13.0) and (key > '10') and (key < '20') and (key > '15') and (key < '25')) (type: boolean) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) Map 4 Map Operator Tree: TableScan @@ -469,7 +469,7 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 - Inner Join 1 to 2 + Inner Join 0 to 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) @@ -477,7 +477,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col0 (type: string) + expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out index ef8c674..e4c3d2b 100644 --- ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out +++ ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out @@ -30,7 +30,7 @@ POSTHOOK: query: create table t4 (id int, key string, value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@t4 -Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from t1 full outer join t2 on t1.id=t2.id join t3 on t2.id=t3.id where t3.id=20 PREHOOK: type: QUERY POSTHOOK: query: explain select * from t1 full outer join t2 on t1.id=t2.id join t3 on t2.id=t3.id where t3.id=20 @@ -128,7 +128,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from t1 join t2 on (t1.id=t2.id) left outer join t3 on (t2.id=t3.id) where t2.id=20 PREHOOK: type: QUERY POSTHOOK: query: explain select * from t1 join t2 on (t1.id=t2.id) left outer join t3 on (t2.id=t3.id) where t2.id=20 @@ -222,7 +222,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Work 'Reducer 2' is a cross product PREHOOK: query: explain select * from t1 join t2 on (t1.id=t2.id) left outer join t3 on (t1.id=t3.id) where t2.id=20 PREHOOK: type: QUERY POSTHOOK: query: explain select * from t1 join t2 on (t1.id=t2.id) left outer join t3 on (t1.id=t3.id) where t2.id=20 diff --git ql/src/test/results/clientpositive/spark/skewjoin.q.out ql/src/test/results/clientpositive/spark/skewjoin.q.out index 1475995..6f9e674 100644 --- ql/src/test/results/clientpositive/spark/skewjoin.q.out +++ ql/src/test/results/clientpositive/spark/skewjoin.q.out @@ -847,9 +847,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + UDFToDouble(1)) (type: double) + key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) sort order: ++ - Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + UDFToDouble(1)) (type: double) + Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reducer 2 @@ -860,7 +860,7 @@ STAGE PLANS: handleSkewJoin: true keys: 0 _col0 (type: string), UDFToDouble(substring(_col1, 5)) (type: double) - 1 _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + UDFToDouble(1)) (type: double) + 1 _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) outputColumnNames: _col2, _col3 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Select Operator diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out index 4b392ba..75422ec 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out @@ -46,8 +46,9 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket_3 -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product -Warning: Shuffle Join JOIN[20][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 6' is a cross product +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[18][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 6' is a cross product +Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select * from (select a.key from smb_bucket_1 a join smb_bucket_2 b on (a.key = b.key) where a.key = 5) t1 left outer join (select c.key from smb_bucket_2 c join smb_bucket_3 d on (c.key = d.key) where c.key=5) t2 on (t1.key=t2.key) where t2.key=5 PREHOOK: type: QUERY @@ -64,7 +65,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 6 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -129,9 +130,7 @@ STAGE PLANS: 1 Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 5 (type: int) - sort order: + - Map-reduce partition columns: 5 (type: int) + sort order: Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: @@ -139,8 +138,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 5 (type: int) - 1 5 (type: int) + 0 + 1 Statistics: Num rows: 31 Data size: 129 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 5 (type: int), 5 (type: int) @@ -163,9 +162,7 @@ STAGE PLANS: 1 Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 5 (type: int) - sort order: + - Map-reduce partition columns: 5 (type: int) + sort order: Statistics: Num rows: 29 Data size: 118 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 @@ -174,8 +171,9 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[32][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[29][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[31][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[30][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: -- explain -- select * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key left outer join src c on a.key=c.value @@ -245,8 +243,8 @@ STAGE PLANS: Statistics: Num rows: 28 Data size: 114 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 5 (type: int) - 1 5 (type: int) + 0 + 1 Local Work: Map Reduce Local Work Map 3 @@ -293,8 +291,8 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 5 (type: int) - 1 5 (type: int) + 0 + 1 input vertices: 0 Map 1 Statistics: Num rows: 31 Data size: 129 Basic stats: COMPLETE Column stats: NONE @@ -318,8 +316,9 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[32][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[29][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[31][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[30][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: select * from (select a.key from smb_bucket_1 a join smb_bucket_2 b on (a.key = b.key) where a.key = 5) t1 left outer join (select c.key from smb_bucket_2 c join smb_bucket_3 d on (c.key = d.key) where c.key=5) t2 on (t1.key=t2.key) where t2.key=5 PREHOOK: type: QUERY PREHOOK: Input: default@smb_bucket_1 diff --git ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out index bbf24fd..8438176 100644 --- ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out +++ ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out @@ -77,7 +77,7 @@ SELECT 1, key, count(1) FROM T1 GROUP BY 1, key PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -Operator:GBY_3 +Operator:GBY_2 Table:default@t1 Keys:key @@ -90,7 +90,7 @@ PREHOOK: query: SELECT key, 1, val, count(1) FROM T1 GROUP BY key, 1, val PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -Operator:GBY_3 +Operator:GBY_2 Table:default@t1 Keys:key,val @@ -104,7 +104,7 @@ PREHOOK: query: SELECT key, 1, val, 2, count(1) FROM T1 GROUP BY key, 1, val, 2 PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -Operator:GBY_3 +Operator:GBY_2 Table:default@t1 Keys:key,val @@ -413,7 +413,7 @@ PREHOOK: Input: default@t2 #### A masked pattern was here #### Operator:JOIN_8 Table:default@t1 -Keys:val,key +Keys:key Table:default@t2 Keys:key @@ -439,7 +439,7 @@ Operator:JOIN_8 Table:default@t1 Keys:key Table:default@t2 -Keys:val,key +Keys:key PREHOOK: query: -- no mapping on functions SELECT * @@ -474,7 +474,7 @@ PREHOOK: Input: default@t2 #### A masked pattern was here #### Operator:JOIN_8 Table:default@t1 -Keys:val,key +Keys:key Table:default@t2 Keys:key @@ -505,7 +505,7 @@ PREHOOK: Input: default@t3 #### A masked pattern was here #### Operator:JOIN_8 Table:default@t1 -Keys:val,key +Keys:key Table:default@t2 Keys:key @@ -543,7 +543,7 @@ PREHOOK: Input: default@t3 #### A masked pattern was here #### Operator:JOIN_8 Table:default@t1 -Keys:val,key +Keys:key Table:default@t2 Keys:key diff --git ql/src/test/results/clientpositive/spark/union_view.q.out ql/src/test/results/clientpositive/spark/union_view.q.out index 3372afb..7ac641f 100644 --- ql/src/test/results/clientpositive/spark/union_view.q.out +++ ql/src/test/results/clientpositive/spark/union_view.q.out @@ -541,14 +541,14 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col1, _col2 + expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) sort order: + Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Map 3 Map Operator Tree: TableScan @@ -563,14 +563,14 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col1, _col2 + expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) sort order: + Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Map 4 Map Operator Tree: TableScan @@ -585,18 +585,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col1, _col2 + expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) sort order: + Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Reducer 2 Reduce Operator Tree: Select Operator - expressions: 86 (type: int), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out index 8b3d353..4ace18b 100644 --- ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out +++ ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out @@ -205,18 +205,18 @@ STAGE PLANS: predicate: ((l_shipmode = 'AIR') and (l_linenumber = 1) and l_orderkey is not null) (type: boolean) Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: l_orderkey (type: int) - outputColumnNames: _col0 + expressions: l_orderkey (type: int), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: int), 1 (type: int) + keys: _col0 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: int), 1 (type: int) - 1 _col0 (type: int), 1 (type: int) + 0 _col0 (type: int), _col3 (type: int) + 1 _col0 (type: int), _col1 (type: int) Local Work: Map Reduce Local Work Map 3 @@ -265,15 +265,15 @@ STAGE PLANS: predicate: ((l_linenumber = 1) and l_partkey is not null and l_orderkey is not null) (type: boolean) Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) - outputColumnNames: _col0, _col1, _col2 + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col0 (type: int), 1 (type: int) - 1 _col0 (type: int), 1 (type: int) + 0 _col0 (type: int), _col3 (type: int) + 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col1, _col2 input vertices: 1 Map 2 diff --git ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out index 50134d9..1d13e69 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -65,7 +65,7 @@ POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [(alltypesorc)alltypesorc POSTHOOK: Lineage: small_alltypesorc3a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc @@ -81,13 +81,13 @@ POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [] PREHOOK: query: select * from small_alltypesorc1a PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc1a diff --git ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out index dba7cbd..d6c8f69 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -54,7 +54,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default POSTHOOK: Output: default@small_alltypesorc3a -POSTHOOK: Lineage: small_alltypesorc3a.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.cbigint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc3a.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] @@ -76,12 +76,12 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default POSTHOOK: Output: default@small_alltypesorc4a -POSTHOOK: Lineage: small_alltypesorc4a.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cbigint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] diff --git ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out index 1c3b7a6..5a1453f 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -61,7 +61,7 @@ POSTHOOK: Lineage: small_alltypesorc3a.cdouble SIMPLE [(alltypesorc)alltypesorc. POSTHOOK: Lineage: small_alltypesorc3a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [] POSTHOOK: Lineage: small_alltypesorc3a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] @@ -81,9 +81,9 @@ POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] diff --git ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out index cc1db38..7540d56 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2b.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2b.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2b.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2b.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2b.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -65,7 +65,7 @@ POSTHOOK: Lineage: small_alltypesorc3b.cstring1 SIMPLE [(alltypesorc)alltypesorc POSTHOOK: Lineage: small_alltypesorc3b.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3b.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3b.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3b.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3b.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc @@ -81,13 +81,13 @@ POSTHOOK: Lineage: small_alltypesorc4b.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4b.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4b.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4b.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4b.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4b.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4b.ctinyint SIMPLE [] PREHOOK: query: select * from small_alltypesorc1b PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc1b diff --git ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out index 8c065f2..c30984d 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out @@ -596,7 +596,7 @@ STAGE PLANS: Spark HashTable Sink Operator keys: 0 UDFToLong(_col1) (type: bigint) - 1 (_col0 pmod UDFToLong(8)) (type: bigint) + 1 (_col0 pmod 8) (type: bigint) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -637,7 +637,7 @@ STAGE PLANS: Left Outer Join0 to 1 keys: 0 UDFToLong(_col1) (type: bigint) - 1 (_col0 pmod UDFToLong(8)) (type: bigint) + 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 input vertices: 1 Map 3 @@ -1300,7 +1300,7 @@ STAGE PLANS: Spark HashTable Sink Operator keys: 0 UDFToLong(_col1) (type: bigint) - 1 (_col0 pmod UDFToLong(8)) (type: bigint) + 1 (_col0 pmod 8) (type: bigint) Execution mode: vectorized Local Work: Map Reduce Local Work @@ -1341,7 +1341,7 @@ STAGE PLANS: Left Outer Join0 to 1 keys: 0 UDFToLong(_col1) (type: bigint) - 1 (_col0 pmod UDFToLong(8)) (type: bigint) + 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 input vertices: 1 Map 3 diff --git ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out index 7caa50d..78b5a7e 100644 --- ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out +++ ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out @@ -2345,7 +2345,7 @@ STAGE PLANS: alias: alltypesorc Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(ctimestamp1) <> 0.0) and (((-257 <> UDFToInteger(ctinyint)) and cstring1 regexp '.*ss' and (-3.0 < UDFToDouble(ctimestamp1)) and cboolean2 is not null) or (UDFToDouble(ctimestamp2) = -5.0) or ((UDFToDouble(ctimestamp1) < 0.0) and (cstring2 like '%b%')) or (cdouble = UDFToDouble(cint)) or (cboolean1 is null and (cfloat < UDFToFloat(cint))))) (type: boolean) + predicate: ((UDFToDouble(ctimestamp1) <> 0.0) and (((-257 <> UDFToInteger(ctinyint)) and cboolean2 is not null and cstring1 regexp '.*ss' and (-3.0 < UDFToDouble(ctimestamp1))) or (UDFToDouble(ctimestamp2) = -5.0) or ((UDFToDouble(ctimestamp1) < 0.0) and (cstring2 like '%b%')) or (cdouble = UDFToDouble(cint)) or (cboolean1 is null and (cfloat < UDFToFloat(cint))))) (type: boolean) Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cstring1 (type: string), ctimestamp1 (type: timestamp), cint (type: int), csmallint (type: smallint), ctinyint (type: tinyint), cfloat (type: float), cdouble (type: double) @@ -2678,7 +2678,7 @@ STAGE PLANS: alias: alltypesorc Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((cdouble < UDFToDouble(csmallint)) and (cboolean2 = cboolean1) and (UDFToDouble(cbigint) <= -863.257)) or ((cint >= -257) and (cboolean1 >= 1) and cstring1 is not null) or cstring2 regexp 'b' or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null)) and cboolean1 is not null) (type: boolean) + predicate: ((((cdouble < UDFToDouble(csmallint)) and (cboolean2 = cboolean1) and (UDFToDouble(cbigint) <= -863.257)) or ((cint >= -257) and cstring1 is not null and (cboolean1 >= 1)) or cstring2 regexp 'b' or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null)) and cboolean1 is not null) (type: boolean) Statistics: Num rows: 10239 Data size: 314333 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cbigint (type: bigint), cint (type: int), cdouble (type: double), ctinyint (type: tinyint), csmallint (type: smallint) diff --git ql/src/test/results/clientpositive/subquery_notin.q.out ql/src/test/results/clientpositive/subquery_notin.q.out index c600b7f..fed1f89 100644 --- ql/src/test/results/clientpositive/subquery_notin.q.out +++ ql/src/test/results/clientpositive/subquery_notin.q.out @@ -29,32 +29,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((key > '2') and key is null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (_col0 = 0) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false table: @@ -79,7 +79,7 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Operator Tree: Join Operator condition map: @@ -285,7 +285,7 @@ POSTHOOK: Input: default@src 199 val_199 199 val_199 2 val_2 -Warning: Shuffle Join JOIN[27][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- non agg, corr explain select p_mfgr, b.p_name, p_size @@ -530,7 +530,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[27][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select p_mfgr, b.p_name, p_size from part b where b.p_name not in @@ -569,7 +569,7 @@ Manufacturer#4 almond azure aquamarine papaya violet 12 Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#5 almond aquamarine dodger light gainsboro 46 Manufacturer#5 almond azure blanched chiffon midnight 23 -Warning: Shuffle Join JOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- agg, non corr explain select p_name, p_size @@ -851,7 +851,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select p_name, p_size from part where part.p_size not in @@ -898,7 +898,7 @@ almond aquamarine sandy cyan gainsboro 18 almond aquamarine yellow dodger mint 7 almond azure aquamarine papaya violet 12 almond azure blanched chiffon midnight 23 -Warning: Shuffle Join JOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- agg, corr explain select p_mfgr, p_name, p_size @@ -1212,7 +1212,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[36][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select p_mfgr, p_name, p_size from part b where b.p_size not in (select min(p_size) @@ -1253,7 +1253,7 @@ Manufacturer#5 almond antique medium spring khaki 6 Manufacturer#5 almond azure blanched chiffon midnight 23 Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#5 almond aquamarine dodger light gainsboro 46 -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- non agg, non corr, Group By in Parent Query select li.l_partkey, count(*) from lineitem li @@ -1452,7 +1452,7 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@t1_v POSTHOOK: Output: database:default POSTHOOK: Output: default@T2_v -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) @@ -1556,17 +1556,17 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < '11') and (CASE WHEN ((key > '104')) THEN (null) ELSE (key) END < '11')) (type: boolean) - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + predicate: ((key < '11') and CASE WHEN ((key > '104')) THEN (null) ELSE ((key < '11')) END) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: CASE WHEN ((key > '104')) THEN (null) ELSE (key) END (type: string) outputColumnNames: _col0 - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1597,7 +1597,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out index 5114296..793b8be 100644 --- ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out +++ ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out @@ -78,7 +78,7 @@ STAGE PLANS: TableScan Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Operator Tree: Join Operator condition map: @@ -149,32 +149,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((key > '12') and key is null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (_col0 = 0) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false table: @@ -188,7 +188,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: -- non agg, corr explain select b.p_mfgr, min(p_retailprice) @@ -445,7 +445,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: select b.p_mfgr, min(p_retailprice) from part b group by b.p_mfgr @@ -620,24 +620,24 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_retailprice (type: double) - outputColumnNames: p_retailprice + outputColumnNames: _col1 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: max(p_retailprice), min(p_retailprice) + aggregations: max(_col1), min(_col1) keys: null (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: null (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: null (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double), _col2 (type: double) Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0), min(VALUE._col1) - keys: null (type: string) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out index 7853737..f9b1bfe 100644 --- ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out +++ ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out @@ -781,7 +781,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[27][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- non agg, corr explain select p_mfgr, b.p_name, p_size diff --git ql/src/test/results/clientpositive/subquery_views.q.out ql/src/test/results/clientpositive/subquery_views.q.out index fab919d..e10b256 100644 --- ql/src/test/results/clientpositive/subquery_views.q.out +++ ql/src/test/results/clientpositive/subquery_views.q.out @@ -111,8 +111,8 @@ where `b`.`key` not in from `default`.`src` `a` where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_11' ), tableType:VIRTUAL_VIEW) -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -Warning: Shuffle Join JOIN[42][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[40][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product PREHOOK: query: explain select * from cv2 where cv2.key in (select key from cv2 c where c.key < '11') @@ -420,8 +420,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -Warning: Shuffle Join JOIN[42][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[40][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product PREHOOK: query: select * from cv2 where cv2.key in (select key from cv2 c where c.key < '11') PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/table_access_keys_stats.q.out ql/src/test/results/clientpositive/table_access_keys_stats.q.out index bbf24fd..8438176 100644 --- ql/src/test/results/clientpositive/table_access_keys_stats.q.out +++ ql/src/test/results/clientpositive/table_access_keys_stats.q.out @@ -77,7 +77,7 @@ SELECT 1, key, count(1) FROM T1 GROUP BY 1, key PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -Operator:GBY_3 +Operator:GBY_2 Table:default@t1 Keys:key @@ -90,7 +90,7 @@ PREHOOK: query: SELECT key, 1, val, count(1) FROM T1 GROUP BY key, 1, val PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -Operator:GBY_3 +Operator:GBY_2 Table:default@t1 Keys:key,val @@ -104,7 +104,7 @@ PREHOOK: query: SELECT key, 1, val, 2, count(1) FROM T1 GROUP BY key, 1, val, 2 PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -Operator:GBY_3 +Operator:GBY_2 Table:default@t1 Keys:key,val @@ -413,7 +413,7 @@ PREHOOK: Input: default@t2 #### A masked pattern was here #### Operator:JOIN_8 Table:default@t1 -Keys:val,key +Keys:key Table:default@t2 Keys:key @@ -439,7 +439,7 @@ Operator:JOIN_8 Table:default@t1 Keys:key Table:default@t2 -Keys:val,key +Keys:key PREHOOK: query: -- no mapping on functions SELECT * @@ -474,7 +474,7 @@ PREHOOK: Input: default@t2 #### A masked pattern was here #### Operator:JOIN_8 Table:default@t1 -Keys:val,key +Keys:key Table:default@t2 Keys:key @@ -505,7 +505,7 @@ PREHOOK: Input: default@t3 #### A masked pattern was here #### Operator:JOIN_8 Table:default@t1 -Keys:val,key +Keys:key Table:default@t2 Keys:key @@ -543,7 +543,7 @@ PREHOOK: Input: default@t3 #### A masked pattern was here #### Operator:JOIN_8 Table:default@t1 -Keys:val,key +Keys:key Table:default@t2 Keys:key diff --git ql/src/test/results/clientpositive/tez/auto_join_filters.q.out ql/src/test/results/clientpositive/tez/auto_join_filters.q.out index 1559d4b..36f719b 100644 --- ql/src/test/results/clientpositive/tez/auto_join_filters.q.out +++ ql/src/test/results/clientpositive/tez/auto_join_filters.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -300,7 +300,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in2.txt' into table sm POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_input2 -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/tez/auto_join_nulls.q.out ql/src/test/results/clientpositive/tez/auto_join_nulls.q.out index 5b68bb7..5984e8f 100644 --- ql/src/test/results/clientpositive/tez/auto_join_nulls.q.out +++ ql/src/test/results/clientpositive/tez/auto_join_nulls.q.out @@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE my POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@myinput1 -Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -24,7 +24,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out index dfcf7ea..3b59cb8 100644 --- ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out +++ ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out @@ -138,7 +138,7 @@ POSTHOOK: query: load data local inpath '../../data/files/smallsrcsortbucket3out POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@bucket_medium@ds=2008-04-08 -Warning: Map Join MAPJOIN[35][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 3' is a cross product PREHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY POSTHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key @@ -559,7 +559,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[35][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[34][bigTable=?] in task 'Map 3' is a cross product PREHOOK: query: select count(*) FROM bucket_small a JOIN bucket_medium b ON a.key = b.key JOIN bucket_big c ON c.key = b.key JOIN bucket_medium d ON c.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@bucket_big diff --git ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out index 636410a..0f3bd63 100644 --- ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out +++ ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out @@ -261,7 +261,7 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_18] (rows=5 width=22) + Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) Conds:RS_8._col3, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_8] @@ -269,7 +269,7 @@ Stage-0 Select Operator [SEL_2] (rows=5 width=20) Output:["_col0","_col1","_col2","_col3"] Filter Operator [FIL_15] (rows=5 width=20) - predicate:(((dimid = 100) = true) and (dimid <> 100) and (dimid = 100) is not null) + predicate:(((dimid = 100) = true) and (dimid <> 100)) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] <-Map 3 [SIMPLE_EDGE] @@ -279,8 +279,8 @@ Stage-0 Output:["_col0","_col1"],keys:_col0, _col1 Select Operator [SEL_5] (rows=2 width=3) Output:["_col0","_col1"] - Filter Operator [FIL_17] (rows=2 width=3) - predicate:(((id = 100) = true) and (id <> 100) and (id = 100) is not null) + Filter Operator [FIL_16] (rows=2 width=3) + predicate:(((id = 100) = true) and (id <> 100)) TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] @@ -309,7 +309,7 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_18] (rows=2 width=22) + Merge Join Operator [MERGEJOIN_17] (rows=2 width=22) Conds:RS_8._col3, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_8] @@ -317,7 +317,7 @@ Stage-0 Select Operator [SEL_2] (rows=2 width=20) Output:["_col0","_col1","_col2","_col3"] Filter Operator [FIL_15] (rows=2 width=20) - predicate:((dimid) IN (100, 200) and ((dimid = 100) = true) and (dimid = 100) is not null) + predicate:((dimid) IN (100, 200) and ((dimid = 100) = true)) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] <-Map 3 [SIMPLE_EDGE] @@ -327,8 +327,8 @@ Stage-0 Output:["_col0","_col1"],keys:_col0, _col1 Select Operator [SEL_5] (rows=1 width=3) Output:["_col0","_col1"] - Filter Operator [FIL_17] (rows=1 width=3) - predicate:((id) IN (100, 200) and ((id = 100) = true) and (id = 100) is not null) + Filter Operator [FIL_16] (rows=1 width=3) + predicate:((id) IN (100, 200) and ((id = 100) = true)) TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] @@ -359,25 +359,26 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_18] (rows=2 width=22) - Conds:RS_8.200, true=RS_9.200, false(Left Semi),Output:["_col0","_col1","_col2"] + Merge Join Operator [MERGEJOIN_17] (rows=2 width=22) + Conds:RS_8.200, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_8] PartitionCols:200, true Select Operator [SEL_2] (rows=2 width=20) Output:["_col0","_col1","_col2"] Filter Operator [FIL_15] (rows=2 width=20) - predicate:(((dimid = 100) = true) and (dimid = 200) and (dimid = 100) is not null) + predicate:(((dimid = 100) = true) and (dimid = 200)) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] <-Map 3 [SIMPLE_EDGE] SHUFFLE [RS_9] - PartitionCols:200, false + PartitionCols:_col0, _col1 Group By Operator [GBY_7] (rows=1 width=3) - Output:["_col0","_col1"],keys:200, false + Output:["_col0","_col1"],keys:_col0, _col1 Select Operator [SEL_5] (rows=1 width=3) - Filter Operator [FIL_17] (rows=1 width=3) - predicate:(((id = 100) = true) and (id = 200) and (id = 100) is not null) + Output:["_col0","_col1"] + Filter Operator [FIL_16] (rows=1 width=3) + predicate:(((id = 100) = true) and (id = 200)) TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] @@ -406,25 +407,26 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_18] (rows=2 width=22) - Conds:RS_8.100, true=RS_9.100, true(Left Semi),Output:["_col0","_col1","_col2"] + Merge Join Operator [MERGEJOIN_17] (rows=2 width=22) + Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_8] PartitionCols:100, true Select Operator [SEL_2] (rows=2 width=20) Output:["_col0","_col1","_col2"] Filter Operator [FIL_15] (rows=2 width=20) - predicate:(((dimid = 100) = true) and (dimid = 100) and (dimid = 100) is not null) + predicate:(((dimid = 100) = true) and (dimid = 100)) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] <-Map 3 [SIMPLE_EDGE] SHUFFLE [RS_9] - PartitionCols:100, true + PartitionCols:_col0, _col1 Group By Operator [GBY_7] (rows=1 width=3) - Output:["_col0","_col1"],keys:100, true + Output:["_col0","_col1"],keys:_col0, _col1 Select Operator [SEL_5] (rows=1 width=3) - Filter Operator [FIL_17] (rows=1 width=3) - predicate:(((id = 100) = true) and (id = 100) and (id = 100) is not null) + Output:["_col0","_col1"] + Filter Operator [FIL_16] (rows=1 width=3) + predicate:(((id = 100) = true) and (id = 100)) TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] @@ -455,7 +457,7 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_12] - Merge Join Operator [MERGEJOIN_18] (rows=5 width=22) + Merge Join Operator [MERGEJOIN_17] (rows=5 width=22) Conds:RS_8._col3, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_8] @@ -463,7 +465,7 @@ Stage-0 Select Operator [SEL_2] (rows=5 width=20) Output:["_col0","_col1","_col2","_col3"] Filter Operator [FIL_15] (rows=5 width=20) - predicate:(((dimid = 100) = true) and dimid is not null and (dimid = 100) is not null) + predicate:(((dimid = 100) = true) and dimid is not null) TableScan [TS_0] (rows=10 width=20) default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"] <-Map 3 [SIMPLE_EDGE] @@ -473,8 +475,8 @@ Stage-0 Output:["_col0","_col1"],keys:_col0, _col1 Select Operator [SEL_5] (rows=2 width=3) Output:["_col0","_col1"] - Filter Operator [FIL_17] (rows=2 width=3) - predicate:(((id = 100) = true) and id is not null and (id = 100) is not null) + Filter Operator [FIL_16] (rows=2 width=3) + predicate:(((id = 100) = true) and id is not null) TableScan [TS_3] (rows=5 width=3) default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"] diff --git ql/src/test/results/clientpositive/tez/cross_join.q.out ql/src/test/results/clientpositive/tez/cross_join.q.out index 0fa801e..9d1fbcc 100644 --- ql/src/test/results/clientpositive/tez/cross_join.q.out +++ ql/src/test/results/clientpositive/tez/cross_join.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- current explain select src.key from src join src src2 PREHOOK: type: QUERY @@ -64,7 +64,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- ansi cross join explain select src.key from src cross join src src2 PREHOOK: type: QUERY @@ -206,7 +206,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: explain select src.key from src join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src join src src2 @@ -266,7 +266,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: explain select src.key from src cross join src src2 PREHOOK: type: QUERY POSTHOOK: query: explain select src.key from src cross join src src2 diff --git ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out index 470590a..bc021f5 100644 --- ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out +++ ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out @@ -32,7 +32,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -99,7 +99,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -204,7 +204,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[27][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -331,8 +331,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -442,7 +442,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 diff --git ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out index 68df37d..b349de5 100644 --- ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out +++ ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out @@ -32,7 +32,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -95,7 +95,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Map 3' is a cross product +Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 3' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -191,7 +191,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -310,8 +310,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 2' is a cross product -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 2' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -413,7 +413,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[31][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Map Join MAPJOIN[30][bigTable=?] in task 'Reducer 2' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 diff --git ql/src/test/results/clientpositive/tez/cte_mat_1.q.out ql/src/test/results/clientpositive/tez/cte_mat_1.q.out index bbe4296..369cd57 100644 --- ql/src/test/results/clientpositive/tez/cte_mat_1.q.out +++ ql/src/test/results/clientpositive/tez/cte_mat_1.q.out @@ -1,3 +1,4 @@ +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -23,21 +24,19 @@ Stage-0 File Output Operator [FS_10] Select Operator [SEL_9] (rows=275 width=10) Output:["_col0"] - Merge Join Operator [MERGEJOIN_15] (rows=275 width=10) - Conds:RS_6.'5'=RS_7.'5'(Inner) + Merge Join Operator [MERGEJOIN_13] (rows=275 width=10) + Conds:(Inner) <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_6] - PartitionCols:'5' Select Operator [SEL_2] (rows=250 width=10) - Filter Operator [FIL_13] (rows=250 width=10) + Filter Operator [FIL_11] (rows=250 width=10) predicate:(key = '5') TableScan [TS_0] (rows=500 width=10) default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] <-Map 3 [SIMPLE_EDGE] SHUFFLE [RS_7] - PartitionCols:'5' Select Operator [SEL_5] (rows=250 width=10) - Filter Operator [FIL_14] (rows=250 width=10) + Filter Operator [FIL_12] (rows=250 width=10) predicate:(key = '5') TableScan [TS_3] (rows=500 width=10) default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] diff --git ql/src/test/results/clientpositive/tez/cte_mat_2.q.out ql/src/test/results/clientpositive/tez/cte_mat_2.q.out index bbe4296..369cd57 100644 --- ql/src/test/results/clientpositive/tez/cte_mat_2.q.out +++ ql/src/test/results/clientpositive/tez/cte_mat_2.q.out @@ -1,3 +1,4 @@ +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain with q1 as (select * from src where key= '5') select a.key @@ -23,21 +24,19 @@ Stage-0 File Output Operator [FS_10] Select Operator [SEL_9] (rows=275 width=10) Output:["_col0"] - Merge Join Operator [MERGEJOIN_15] (rows=275 width=10) - Conds:RS_6.'5'=RS_7.'5'(Inner) + Merge Join Operator [MERGEJOIN_13] (rows=275 width=10) + Conds:(Inner) <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_6] - PartitionCols:'5' Select Operator [SEL_2] (rows=250 width=10) - Filter Operator [FIL_13] (rows=250 width=10) + Filter Operator [FIL_11] (rows=250 width=10) predicate:(key = '5') TableScan [TS_0] (rows=500 width=10) default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] <-Map 3 [SIMPLE_EDGE] SHUFFLE [RS_7] - PartitionCols:'5' Select Operator [SEL_5] (rows=250 width=10) - Filter Operator [FIL_14] (rows=250 width=10) + Filter Operator [FIL_12] (rows=250 width=10) predicate:(key = '5') TableScan [TS_3] (rows=500 width=10) default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] diff --git ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out index 2626768..f6eefb2 100644 --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out @@ -218,7 +218,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -341,7 +340,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -464,7 +462,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -587,7 +584,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -878,7 +874,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) @@ -1174,7 +1169,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds is not null and hr is not null) (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) @@ -1295,7 +1289,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -1418,7 +1411,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -1537,7 +1529,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) @@ -1562,12 +1553,12 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1588,7 +1579,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1660,16 +1651,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + key expressions: (UDFToDouble(_col0) * 2.0) (type: double) sort order: + - Map-reduce partition columns: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + Map-reduce partition columns: (UDFToDouble(_col0) * 2.0) (type: double) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: @@ -1710,7 +1700,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1783,7 +1773,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) @@ -1808,9 +1797,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: @@ -1819,7 +1808,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1891,16 +1880,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + key expressions: (UDFToDouble(_col0) * 2.0) (type: double) sort order: + - Map-reduce partition columns: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + Map-reduce partition columns: (UDFToDouble(_col0) * 2.0) (type: double) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: @@ -1926,7 +1914,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -2012,16 +2000,15 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + key expressions: UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) sort order: + - Map-reduce partition columns: UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + Map-reduce partition columns: UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: @@ -2062,7 +2049,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + 0 UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) 1 UDFToString(_col0) (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -2151,22 +2138,21 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: '2008-04-08' (type: string) @@ -2174,9 +2160,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '2008-04-08' (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: @@ -2186,7 +2172,7 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 '2008-04-08' (type: string) - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -2213,7 +2199,7 @@ STAGE PLANS: Reducer 5 Reduce Operator Tree: Group By Operator - keys: '2008-04-08' (type: string) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE @@ -2224,21 +2210,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: '2008-04-08' (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Dynamic Partitioning Event Operator - Target column: ds (string) - Target Input: srcpart - Partition key expr: ds - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -2251,16 +2222,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -2276,7 +2243,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- non-equi join EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY @@ -2371,7 +2338,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -2553,7 +2520,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -3174,7 +3140,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -3372,7 +3337,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -3574,7 +3538,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: ds (type: string) @@ -3608,7 +3571,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: ds (type: string) @@ -3833,7 +3795,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -3967,7 +3928,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -4415,7 +4375,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -4523,7 +4482,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) @@ -4534,7 +4492,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) input vertices: 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -4562,12 +4520,12 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -4642,7 +4600,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: hr is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) @@ -4652,7 +4609,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) input vertices: 1 Map 3 @@ -4768,45 +4725,29 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 '2008-04-08' (type: string) - input vertices: - 1 Reducer 4 - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 3 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Map 2 Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: '2008-04-08' (type: string) @@ -4814,11 +4755,39 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '2008-04-08' (type: string) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Reducer 3 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 '2008-04-08' (type: string) + input vertices: + 0 Map 1 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -4832,35 +4801,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Reduce Operator Tree: - Group By Operator - keys: '2008-04-08' (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '2008-04-08' (type: string) - sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Dynamic Partitioning Event Operator - Target column: ds (string) - Target Input: srcpart - Partition key expr: ds - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -4873,16 +4813,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -4921,7 +4857,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) @@ -5474,7 +5409,6 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) diff --git ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out index 71b7ee3..08fc33e 100644 --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out @@ -561,7 +561,7 @@ bar baz baz baz -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: EXPLAIN SELECT agg.amount FROM agg_01 agg, dim_shops d1 @@ -634,7 +634,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT agg.amount FROM agg_01 agg, dim_shops d1 diff --git ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out index 2f88148..6689394 100644 --- ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out +++ ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out @@ -2409,29 +2409,29 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over1k - Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (s = 'foo') (type: boolean) - Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), t (type: tinyint), i (type: int) - outputColumnNames: _col0, _col1, _col2, _col4, _col5 - Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'foo' (type: string), _col4 (type: tinyint), _col5 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: 'foo' (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), 'foo' (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2485,19 +2485,19 @@ STAGE PLANS: predicate: (t = 27) (type: boolean) Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), i (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col5 + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27 (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string), 27 (type: tinyint), _col5 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: _col3 (type: string), 27 (type: tinyint), _col5 (type: int) + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), 27 (type: tinyint), KEY._col5 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2556,19 +2556,19 @@ STAGE PLANS: predicate: (i = 100) (type: boolean) Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), t (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string), _col4 (type: tinyint), 100 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), 100 (type: int) + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), 100 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2627,19 +2627,19 @@ STAGE PLANS: predicate: ((i = 100) and (t = 27)) (type: boolean) Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27 (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string), 27 (type: tinyint), 100 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: _col3 (type: string), 27 (type: tinyint), 100 (type: int) + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), 27 (type: tinyint), 100 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2693,29 +2693,29 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over1k - Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((i = 100) and (s = 'foo')) (type: boolean) - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), t (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col4 - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'foo' (type: string), _col4 (type: tinyint), 100 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: 'foo' (type: string), _col4 (type: tinyint), 100 (type: int) - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), 'foo' (type: string), KEY._col4 (type: tinyint), 100 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2764,29 +2764,29 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over1k - Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((t = 27) and (s = 'foo')) (type: boolean) - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: si (type: smallint), b (type: bigint), f (type: float), i (type: int) - outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27 (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'foo' (type: string), 27 (type: tinyint), _col5 (type: int) + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ - Map-reduce partition columns: 'foo' (type: string), 27 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Reducer 2 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), 'foo' (type: string), 27 (type: tinyint), KEY._col5 (type: int) + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2832,17 +2832,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over1k - Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((i = 100) and (t = 27) and (s = 'foo')) (type: boolean) - Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 107 Data size: 13282 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27 (type: tinyint), 100 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 107 Data size: 13282 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 107 Data size: 13282 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out index 97f59d9..6931398 100644 --- ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out +++ ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out @@ -1613,31 +1613,31 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) - outputColumnNames: _col1, _col2 + outputColumnNames: key, value Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(_col2) - keys: 'day' (type: string), _col1 (type: string) + aggregations: count(value) + keys: key (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'day' (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: 'day' (type: string), _col1 (type: string) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + value expressions: _col1 (type: bigint) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: 'day' (type: string), KEY._col1 (type: string) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int), 'day' (type: string) + expressions: UDFToInteger(_col0) (type: int), UDFToInteger(_col1) (type: int), 'day' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1741,31 +1741,31 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) - outputColumnNames: _col1, _col2 + outputColumnNames: key, value Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(_col2) - keys: 'day' (type: string), _col1 (type: string) + aggregations: count(value) + keys: key (type: string) mode: hash - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'day' (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: 'day' (type: string), _col1 (type: string) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) + value expressions: _col1 (type: bigint) Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: 'day' (type: string), KEY._col1 (type: string) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0, _col1, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int), 'day' (type: string) + expressions: UDFToInteger(_col0) (type: int), UDFToInteger(_col1) (type: int), 'day' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/tez/explainuser_1.q.out ql/src/test/results/clientpositive/tez/explainuser_1.q.out index 965577e..84f2044 100644 --- ql/src/test/results/clientpositive/tez/explainuser_1.q.out +++ ql/src/test/results/clientpositive/tez/explainuser_1.q.out @@ -1808,20 +1808,16 @@ Stage-0 SHUFFLE [RS_12] Group By Operator [GBY_11] (rows=1 width=8) Output:["_col0"],aggregations:["count('2014')"] - Merge Join Operator [MERGEJOIN_20] (rows=400 width=0) - Conds:RS_6._col0=RS_7._col0(Inner) + Merge Join Operator [MERGEJOIN_18] (rows=400 width=0) + Conds:(Inner) <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_6] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=20 width=184) - Output:["_col0"] + Select Operator [SEL_2] (rows=20 width=88) TableScan [TS_0] (rows=20 width=13) default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE <-Map 4 [SIMPLE_EDGE] SHUFFLE [RS_7] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20 width=184) - Output:["_col0"] + Select Operator [SEL_5] (rows=20 width=88) TableScan [TS_3] (rows=20 width=13) default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE @@ -2128,7 +2124,7 @@ Stage-0 SHUFFLE [RS_18] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_31] (rows=13 width=8) - Conds:RS_15._col0, 1=RS_16._col0, 1(Left Semi),Output:["_col1","_col2"] + Conds:RS_15._col0, 1=RS_16._col0, _col1(Left Semi),Output:["_col1","_col2"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col0, 1 @@ -2140,11 +2136,11 @@ Stage-0 default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_partkey","l_suppkey","l_linenumber"] <-Map 4 [SIMPLE_EDGE] SHUFFLE [RS_16] - PartitionCols:_col0, 1 + PartitionCols:_col0, _col1 Group By Operator [GBY_14] (rows=4 width=8) - Output:["_col0","_col1"],keys:_col0, 1 - Select Operator [SEL_5] (rows=14 width=4) - Output:["_col0"] + Output:["_col0","_col1"],keys:_col0, _col1 + Select Operator [SEL_5] (rows=14 width=8) + Output:["_col0","_col1"] Filter Operator [FIL_29] (rows=14 width=96) predicate:((l_shipmode = 'AIR') and (l_linenumber = 1) and l_orderkey is not null) TableScan [TS_3] (rows=100 width=96) @@ -2347,21 +2343,21 @@ Stage-0 Output:["_col0","_col1"] Filter Operator [FIL_21] (rows=1 width=265) predicate:_col3 is null - Merge Join Operator [MERGEJOIN_30] (rows=404 width=265) + Merge Join Operator [MERGEJOIN_29] (rows=404 width=265) Conds:RS_18._col0=RS_19._col0(Left Outer),Output:["_col0","_col1","_col3"] <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 - Select Operator [SEL_13] (rows=166 width=87) + Select Operator [SEL_14] (rows=166 width=87) Output:["_col0"] - Filter Operator [FIL_28] (rows=166 width=87) + Filter Operator [FIL_27] (rows=166 width=87) predicate:(key > '2') - TableScan [TS_11] (rows=500 width=87) + TableScan [TS_12] (rows=500 width=87) default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_29] (rows=500 width=178) + Merge Join Operator [MERGEJOIN_28] (rows=500 width=178) Conds:(Inner),Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_15] @@ -2371,20 +2367,20 @@ Stage-0 default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_16] - Select Operator [SEL_10] (rows=1 width=8) - Filter Operator [FIL_9] (rows=1 width=8) + Select Operator [SEL_11] (rows=1 width=8) + Filter Operator [FIL_10] (rows=1 width=8) predicate:(_col0 = 0) - Group By Operator [GBY_7] (rows=1 width=8) + Group By Operator [GBY_8] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 5 [SIMPLE_EDGE] - SHUFFLE [RS_6] - Group By Operator [GBY_5] (rows=1 width=8) + SHUFFLE [RS_7] + Group By Operator [GBY_6] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_4] (rows=1 width=87) - Filter Operator [FIL_27] (rows=1 width=87) - predicate:((key > '2') and key is null) - TableScan [TS_2] (rows=500 width=87) - default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + Filter Operator [FIL_4] (rows=1 width=4) + predicate:false + Select Operator [SEL_3] (rows=500 width=4) + TableScan [TS_2] (rows=500 width=10) + default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE PREHOOK: query: explain select p_mfgr, b.p_name, p_size from part b @@ -2414,35 +2410,35 @@ Stage-0 limit:-1 Stage-1 Reducer 3 - File Output Operator [FS_23] - Select Operator [SEL_22] (rows=1 width=223) + File Output Operator [FS_22] + Select Operator [SEL_21] (rows=1 width=223) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_21] (rows=1 width=344) + Filter Operator [FIL_20] (rows=1 width=344) predicate:_col4 is null - Merge Join Operator [MERGEJOIN_28] (rows=1 width=344) - Conds:RS_18._col0, _col1=RS_19._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4"] + Merge Join Operator [MERGEJOIN_27] (rows=1 width=344) + Conds:RS_17._col0, _col1=RS_18._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4"] <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_19] + SHUFFLE [RS_18] PartitionCols:_col0, _col1 Select Operator [SEL_13] (rows=8 width=219) Output:["_col0","_col1"] - Filter Operator [FIL_26] (rows=8 width=223) + Filter Operator [FIL_25] (rows=8 width=223) predicate:(p_size < 10) TableScan [TS_11] (rows=26 width=223) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_18] + SHUFFLE [RS_17] PartitionCols:_col0, _col1 - Merge Join Operator [MERGEJOIN_27] (rows=26 width=223) + Merge Join Operator [MERGEJOIN_26] (rows=26 width=223) Conds:(Inner),Output:["_col0","_col1","_col2"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_15] + SHUFFLE [RS_14] Select Operator [SEL_1] (rows=26 width=223) Output:["_col0","_col1","_col2"] TableScan [TS_0] (rows=26 width=223) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_16] + SHUFFLE [RS_15] Select Operator [SEL_10] (rows=1 width=8) Filter Operator [FIL_9] (rows=1 width=8) predicate:(_col0 = 0) @@ -2453,7 +2449,7 @@ Stage-0 Group By Operator [GBY_5] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Select Operator [SEL_4] (rows=1 width=223) - Filter Operator [FIL_25] (rows=1 width=223) + Filter Operator [FIL_24] (rows=1 width=223) predicate:((p_size < 10) and (p_name is null or p_mfgr is null)) TableScan [TS_2] (rows=26 width=223) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] @@ -2488,30 +2484,30 @@ Stage-0 limit:-1 Stage-1 Reducer 4 - File Output Operator [FS_36] - Select Operator [SEL_35] (rows=1 width=125) + File Output Operator [FS_35] + Select Operator [SEL_34] (rows=1 width=125) Output:["_col0","_col1"] <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_34] - Select Operator [SEL_33] (rows=1 width=125) + SHUFFLE [RS_33] + Select Operator [SEL_32] (rows=1 width=125) Output:["_col0","_col1"] - Filter Operator [FIL_32] (rows=1 width=133) + Filter Operator [FIL_31] (rows=1 width=133) predicate:_col3 is null - Merge Join Operator [MERGEJOIN_42] (rows=1 width=133) - Conds:RS_29.UDFToDouble(_col1)=RS_30._col0(Left Outer),Output:["_col0","_col1","_col3"] + Merge Join Operator [MERGEJOIN_41] (rows=1 width=133) + Conds:RS_28.UDFToDouble(_col1)=RS_29._col0(Left Outer),Output:["_col0","_col1","_col3"] <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_29] + SHUFFLE [RS_28] PartitionCols:UDFToDouble(_col1) - Merge Join Operator [MERGEJOIN_41] (rows=26 width=125) + Merge Join Operator [MERGEJOIN_40] (rows=26 width=125) Conds:(Inner),Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] + SHUFFLE [RS_25] Select Operator [SEL_1] (rows=26 width=125) Output:["_col0","_col1"] TableScan [TS_0] (rows=26 width=125) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_size"] <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_27] + SHUFFLE [RS_26] Select Operator [SEL_17] (rows=1 width=8) Filter Operator [FIL_16] (rows=1 width=8) predicate:(_col0 = 0) @@ -2526,12 +2522,12 @@ Stage-0 SHUFFLE [RS_6] Group By Operator [GBY_5] (rows=1 width=0) Output:["_col0"],aggregations:["avg(p_size)"] - Filter Operator [FIL_38] (rows=8 width=4) + Filter Operator [FIL_37] (rows=8 width=4) predicate:(p_size < 10) TableScan [TS_2] (rows=26 width=4) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_30] + SHUFFLE [RS_29] PartitionCols:_col0 Group By Operator [GBY_23] (rows=1 width=8) Output:["_col0"],aggregations:["avg(VALUE._col0)"] @@ -2539,7 +2535,7 @@ Stage-0 SHUFFLE [RS_22] Group By Operator [GBY_21] (rows=1 width=0) Output:["_col0"],aggregations:["avg(p_size)"] - Filter Operator [FIL_40] (rows=8 width=4) + Filter Operator [FIL_39] (rows=8 width=4) predicate:(p_size < 10) TableScan [TS_18] (rows=26 width=4) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] @@ -2580,23 +2576,23 @@ Stage-0 limit:-1 Stage-1 Reducer 5 - File Output Operator [FS_38] - Select Operator [SEL_37] (rows=1 width=106) + File Output Operator [FS_37] + Select Operator [SEL_36] (rows=1 width=106) Output:["_col0","_col1"] <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_36] - Select Operator [SEL_35] (rows=1 width=106) + SHUFFLE [RS_35] + Select Operator [SEL_34] (rows=1 width=106) Output:["_col0","_col1"] - Filter Operator [FIL_34] (rows=1 width=204) + Filter Operator [FIL_33] (rows=1 width=204) predicate:_col3 is null - Merge Join Operator [MERGEJOIN_43] (rows=1 width=204) - Conds:RS_31._col0, _col1=RS_32._col0, _col1(Left Outer),Output:["_col0","_col1","_col3"] + Merge Join Operator [MERGEJOIN_42] (rows=1 width=204) + Conds:RS_30._col0, _col1=RS_31._col0, _col1(Left Outer),Output:["_col0","_col1","_col3"] <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_32] + SHUFFLE [RS_31] PartitionCols:_col0, _col1 Select Operator [SEL_26] (rows=1 width=106) Output:["_col0","_col1"] - Filter Operator [FIL_40] (rows=1 width=114) + Filter Operator [FIL_39] (rows=1 width=114) predicate:((_col2 - _col1) > 600.0) Group By Operator [GBY_24] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 @@ -2605,17 +2601,15 @@ Stage-0 PartitionCols:_col0 Group By Operator [GBY_22] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr - Select Operator [SEL_21] (rows=26 width=106) - Output:["p_mfgr","p_retailprice"] - TableScan [TS_20] (rows=26 width=106) - default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] + TableScan [TS_20] (rows=26 width=106) + default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_31] + SHUFFLE [RS_30] PartitionCols:_col0, _col1 - Merge Join Operator [MERGEJOIN_42] (rows=5 width=106) + Merge Join Operator [MERGEJOIN_41] (rows=5 width=106) Conds:(Inner),Output:["_col0","_col1"] <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_28] + SHUFFLE [RS_27] Group By Operator [GBY_4] (rows=5 width=106) Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 <-Map 1 [SIMPLE_EDGE] @@ -2628,7 +2622,7 @@ Stage-0 TableScan [TS_0] (rows=26 width=106) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_29] + SHUFFLE [RS_28] Select Operator [SEL_19] (rows=1 width=8) Filter Operator [FIL_18] (rows=1 width=8) predicate:(_col0 = 0) @@ -3296,17 +3290,17 @@ Stage-0 limit:-1 Stage-1 Reducer 2 - File Output Operator [FS_9] - Merge Join Operator [MERGEJOIN_10] (rows=250000 width=87) + File Output Operator [FS_8] + Merge Join Operator [MERGEJOIN_9] (rows=250000 width=87) Conds:(Inner),Output:["_col0"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_5] + SHUFFLE [RS_4] Select Operator [SEL_1] (rows=500 width=87) Output:["_col0"] TableScan [TS_0] (rows=500 width=87) default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_6] + SHUFFLE [RS_5] Select Operator [SEL_3] (rows=500 width=4) TableScan [TS_2] (rows=500 width=10) default@src,src,Tbl:COMPLETE,Col:COMPLETE @@ -5870,10 +5864,10 @@ Stage-0 Select Operator [SEL_7] (rows=1 width=33) Output:["_col0","_col1","_col2"] Map Join Operator [MAPJOIN_17] (rows=1 width=33) - Conds:SEL_1.UDFToDouble(_col0)=RS_5.(UDFToDouble(_col0) + UDFToDouble(1))(Left Outer),HybridGraceHashJoin:true,Output:["_col0","_col1","_col2"] + Conds:SEL_1.UDFToDouble(_col0)=RS_5.(UDFToDouble(_col0) + 1.0)(Left Outer),HybridGraceHashJoin:true,Output:["_col0","_col1","_col2"] <-Map 4 [BROADCAST_EDGE] BROADCAST [RS_5] - PartitionCols:(UDFToDouble(_col0) + UDFToDouble(1)) + PartitionCols:(UDFToDouble(_col0) + 1.0) Select Operator [SEL_3] (rows=1 width=30) Output:["_col0"] TableScan [TS_2] (rows=1 width=30) diff --git ql/src/test/results/clientpositive/tez/explainuser_4.q.out ql/src/test/results/clientpositive/tez/explainuser_4.q.out index 661f95f..0b07a29 100644 --- ql/src/test/results/clientpositive/tez/explainuser_4.q.out +++ ql/src/test/results/clientpositive/tez/explainuser_4.q.out @@ -49,7 +49,7 @@ Stage-0 Select Operator [SEL_5] (rows=6144 width=215) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] Filter Operator [FIL_16] (rows=6144 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) + predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) TableScan [TS_3] (rows=12288 width=215) default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] @@ -130,7 +130,7 @@ Stage-0 Select Operator [SEL_5] (rows=6144 width=215) Output:["_col0"] Filter Operator [FIL_18] (rows=6144 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) + predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) TableScan [TS_3] (rows=12288 width=215) default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] @@ -210,7 +210,7 @@ Stage-0 Select Operator [SEL_5] (rows=6144 width=215) Output:["_col0"] Filter Operator [FIL_20] (rows=6144 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) + predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) TableScan [TS_3] (rows=12288 width=215) default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] @@ -281,7 +281,7 @@ Stage-0 Select Operator [SEL_5] (rows=6144 width=215) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] Filter Operator [FIL_16] (rows=6144 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) + predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) TableScan [TS_3] (rows=12288 width=215) default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"] <-Map 1 [CUSTOM_SIMPLE_EDGE] @@ -362,7 +362,7 @@ Stage-0 Select Operator [SEL_5] (rows=6144 width=215) Output:["_col0"] Filter Operator [FIL_18] (rows=6144 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) + predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) TableScan [TS_3] (rows=12288 width=215) default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] <-Map 1 [CUSTOM_SIMPLE_EDGE] @@ -442,7 +442,7 @@ Stage-0 Select Operator [SEL_5] (rows=6144 width=215) Output:["_col0"] Filter Operator [FIL_20] (rows=6144 width=215) - predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null) + predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) TableScan [TS_3] (rows=12288 width=215) default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"] <-Map 1 [CUSTOM_SIMPLE_EDGE] diff --git ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out index 7c22d9a..25bd48f 100644 --- ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out @@ -1250,7 +1250,7 @@ POSTHOOK: Lineage: decimal_mapjoin.cdecimal1 EXPRESSION [(alltypesorc)alltypesor POSTHOOK: Lineage: decimal_mapjoin.cdecimal2 EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1331,7 +1331,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1446,7 +1446,7 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1527,7 +1527,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint diff --git ql/src/test/results/clientpositive/tez/mergejoin.q.out ql/src/test/results/clientpositive/tez/mergejoin.q.out index 299c4db..6f01a6f 100644 --- ql/src/test/results/clientpositive/tez/mergejoin.q.out +++ ql/src/test/results/clientpositive/tez/mergejoin.q.out @@ -2684,6 +2684,7 @@ full outer join PREHOOK: type: QUERY PREHOOK: Input: default@tab PREHOOK: Input: default@tab_part +PREHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: select * from (select * from tab where tab.key = 0)a @@ -2692,6 +2693,7 @@ full outer join POSTHOOK: type: QUERY POSTHOOK: Input: default@tab POSTHOOK: Input: default@tab_part +POSTHOOK: Input: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: select * from (select * from tab where tab.key = 0)a @@ -2715,7 +2717,7 @@ NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08 -Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select * from (select * from tab where tab.key = 0)a join diff --git ql/src/test/results/clientpositive/tez/skewjoin.q.out ql/src/test/results/clientpositive/tez/skewjoin.q.out index 0ee28fb..cba0aac 100644 --- ql/src/test/results/clientpositive/tez/skewjoin.q.out +++ ql/src/test/results/clientpositive/tez/skewjoin.q.out @@ -735,9 +735,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + UDFToDouble(1)) (type: double) + key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) sort order: ++ - Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + UDFToDouble(1)) (type: double) + Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Reducer 2 @@ -747,7 +747,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), UDFToDouble(substring(_col1, 5)) (type: double) - 1 _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + UDFToDouble(1)) (type: double) + 1 _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double) outputColumnNames: _col2, _col3 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Select Operator diff --git ql/src/test/results/clientpositive/tez/tez_dynpart_hashjoin_1.q.out ql/src/test/results/clientpositive/tez/tez_dynpart_hashjoin_1.q.out index eb40bd7..a88c33a 100644 --- ql/src/test/results/clientpositive/tez/tez_dynpart_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/tez/tez_dynpart_hashjoin_1.q.out @@ -53,7 +53,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) @@ -178,7 +178,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -301,7 +301,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -447,7 +447,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) @@ -575,7 +575,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -701,7 +701,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) diff --git ql/src/test/results/clientpositive/tez/tez_self_join.q.out ql/src/test/results/clientpositive/tez/tez_self_join.q.out index f5375b2..52404a0 100644 --- ql/src/test/results/clientpositive/tez/tez_self_join.q.out +++ ql/src/test/results/clientpositive/tez/tez_self_join.q.out @@ -42,6 +42,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@values__tmp__table__2 POSTHOOK: Output: default@tez_self_join2 POSTHOOK: Lineage: tez_self_join2.id1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select s.id2, s.id3 from @@ -90,9 +91,7 @@ STAGE PLANS: outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'ab' (type: string) - sort order: + - Map-reduce partition columns: 'ab' (type: string) + sort order: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col2 (type: string) Map 4 @@ -106,9 +105,7 @@ STAGE PLANS: Select Operator Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: 'ab' (type: string) - sort order: + - Map-reduce partition columns: 'ab' (type: string) + sort order: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Map 5 Map Operator Tree: @@ -133,16 +130,20 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 'ab' (type: string) - 1 'ab' (type: string) + 0 + 1 outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: _col0 (type: int), _col2 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reducer 3 Reduce Operator Tree: Merge Join Operator @@ -151,10 +152,10 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2 + outputColumnNames: _col1 Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 'ab' (type: string), _col2 (type: string) + expressions: 'ab' (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -171,6 +172,7 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select s.id2, s.id3 from ( diff --git ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out index abb0707..a1221a4 100644 --- ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/tez/tez_union_dynamic_partition.q.out @@ -66,42 +66,34 @@ STAGE PLANS: alias: dummy Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 1 (type: int) - outputColumnNames: _col0 + expressions: 1 (type: int), '2014' (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), '2014' (type: string) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.partunion1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partunion1 Map 3 Map Operator Tree: TableScan alias: dummy Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 2 (type: int) - outputColumnNames: _col0 + expressions: 2 (type: int), '2014' (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), '2014' (type: string) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.partunion1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partunion1 Union 2 Vertex: Union 2 diff --git ql/src/test/results/clientpositive/tez/tez_vector_dynpart_hashjoin_1.q.out ql/src/test/results/clientpositive/tez/tez_vector_dynpart_hashjoin_1.q.out index 47699c6..776e02e 100644 --- ql/src/test/results/clientpositive/tez/tez_vector_dynpart_hashjoin_1.q.out +++ ql/src/test/results/clientpositive/tez/tez_vector_dynpart_hashjoin_1.q.out @@ -53,7 +53,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) @@ -178,7 +178,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -301,7 +301,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -448,7 +448,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean) @@ -580,7 +580,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) @@ -710,7 +710,7 @@ STAGE PLANS: alias: a Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (cint BETWEEN 1000000 AND 3000000 and cbigint is not null) (type: boolean) + predicate: (cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000) (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cint (type: int) diff --git ql/src/test/results/clientpositive/tez/union_fast_stats.q.out ql/src/test/results/clientpositive/tez/union_fast_stats.q.out index 578205e..0cc0b9c 100644 --- ql/src/test/results/clientpositive/tez/union_fast_stats.q.out +++ ql/src/test/results/clientpositive/tez/union_fast_stats.q.out @@ -55,7 +55,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -83,7 +83,7 @@ POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [(alltypesorc)alltypesorc POSTHOOK: Lineage: small_alltypesorc3a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc @@ -99,13 +99,13 @@ POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from (select * from (select * from small_alltypesorc1a) sq1 union all @@ -388,7 +388,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -416,7 +416,7 @@ POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [(alltypesorc)alltypesorc POSTHOOK: Lineage: small_alltypesorc3a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc @@ -432,13 +432,13 @@ POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from (select * from (select * from small_alltypesorc1a) sq1 union all diff --git ql/src/test/results/clientpositive/tez/vector_between_columns.q.out ql/src/test/results/clientpositive/tez/vector_between_columns.q.out index d548364..8a9978b 100644 --- ql/src/test/results/clientpositive/tez/vector_between_columns.q.out +++ ql/src/test/results/clientpositive/tez/vector_between_columns.q.out @@ -70,7 +70,7 @@ POSTHOOK: Output: default@TINT POSTHOOK: Lineage: tint.cint SIMPLE [(tint_txt)tint_txt.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: tint.rnum SIMPLE [(tint_txt)tint_txt.FieldSchema(name:rnum, type:int, comment:null), ] tint_txt.rnum tint_txt.cint -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: explain select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where tint.cint between tsint.csint and tsint.csint PREHOOK: type: QUERY @@ -144,7 +144,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where tint.cint between tsint.csint and tsint.csint PREHOOK: type: QUERY PREHOOK: Input: default@tint diff --git ql/src/test/results/clientpositive/tez/vector_binary_join_groupby.q.out ql/src/test/results/clientpositive/tez/vector_binary_join_groupby.q.out index 2bf93e3..8cbb4b1 100644 --- ql/src/test/results/clientpositive/tez/vector_binary_join_groupby.q.out +++ ql/src/test/results/clientpositive/tez/vector_binary_join_groupby.q.out @@ -194,7 +194,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT sum(hash(*)) FROM hundredorc t1 JOIN hundredorc t2 ON t2.bin = t2.bin PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/tez/vector_coalesce.q.out ql/src/test/results/clientpositive/tez/vector_coalesce.q.out index bb67008..e705e60 100644 --- ql/src/test/results/clientpositive/tez/vector_coalesce.q.out +++ ql/src/test/results/clientpositive/tez/vector_coalesce.q.out @@ -48,7 +48,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: null (type: double), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: smallint), KEY.reducesinkkey5 (type: string) + expressions: KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: smallint), KEY.reducesinkkey5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Limit @@ -140,7 +140,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: null (type: tinyint), KEY.reducesinkkey1 (type: double), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: double) + expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: double) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Limit @@ -230,7 +230,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: null (type: float), null (type: bigint), 0.0 (type: float) + expressions: KEY.reducesinkkey0 (type: float), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE Limit @@ -412,7 +412,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: null (type: float), null (type: bigint), null (type: float) + expressions: KEY.reducesinkkey0 (type: float), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey0 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE Limit @@ -481,7 +481,7 @@ STAGE PLANS: Filter Operator predicate: cbigint is null (type: boolean) Select Operator - expressions: null (type: bigint), ctinyint (type: tinyint), COALESCE(null,ctinyint) (type: bigint) + expressions: null (type: bigint), ctinyint (type: tinyint), COALESCE(null,ctinyint) (type: tinyint) outputColumnNames: _col0, _col1, _col2 Limit Number of rows: 10 diff --git ql/src/test/results/clientpositive/tez/vector_date_1.q.out ql/src/test/results/clientpositive/tez/vector_date_1.q.out index a27edcb..9f31820 100644 --- ql/src/test/results/clientpositive/tez/vector_date_1.q.out +++ ql/src/test/results/clientpositive/tez/vector_date_1.q.out @@ -665,7 +665,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 2001-01-01 (type: date), VALUE._col0 (type: date) + expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: date) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 74 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/tez/vector_decimal_2.q.out ql/src/test/results/clientpositive/tez/vector_decimal_2.q.out index 701f0b7..292fb88 100644 --- ql/src/test/results/clientpositive/tez/vector_decimal_2.q.out +++ ql/src/test/results/clientpositive/tez/vector_decimal_2.q.out @@ -1048,17 +1048,20 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3.14 (type: decimal(4,2)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3.14 (type: decimal(3,2)) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(4,2)) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3.14 (type: decimal(4,2)) + expressions: VALUE._col0 (type: decimal(4,2)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1108,17 +1111,20 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3.14 (type: decimal(4,2)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3.14 (type: decimal(3,2)) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(4,2)) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3.14 (type: decimal(4,2)) + expressions: VALUE._col0 (type: decimal(4,2)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1168,17 +1174,20 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 1355944339.1234567 (type: decimal(30,8)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 1355944339.1234567 (type: decimal(17,7)) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(30,8)) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 1355944339.1234567 (type: decimal(30,8)) + expressions: VALUE._col0 (type: decimal(30,8)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1228,17 +1237,20 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 1 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 1 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 1 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1279,17 +1291,20 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 1 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 1 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 1 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1339,17 +1354,20 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1399,17 +1417,20 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1459,17 +1480,20 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1519,17 +1543,20 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1579,17 +1606,20 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 1 (type: decimal(20,19)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 1 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(20,19)) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 1 (type: decimal(20,19)) + expressions: VALUE._col0 (type: decimal(20,19)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1649,7 +1679,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 0.99999999999999999999 (type: decimal(20,20)) + expressions: KEY.reducesinkkey0 (type: decimal(20,20)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git ql/src/test/results/clientpositive/tez/vector_decimal_round_2.q.out ql/src/test/results/clientpositive/tez/vector_decimal_round_2.q.out index a7b8385..3b6d42d 100644 --- ql/src/test/results/clientpositive/tez/vector_decimal_round_2.q.out +++ ql/src/test/results/clientpositive/tez/vector_decimal_round_2.q.out @@ -461,20 +461,20 @@ STAGE PLANS: alias: decimal_tbl_4_orc Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: round(pos, 9) (type: decimal(30,9)), round(neg, 9) (type: decimal(30,9)) - outputColumnNames: _col0, _col1 + expressions: round(pos, 9) (type: decimal(30,9)), round(neg, 9) (type: decimal(30,9)), 1809242.315111134 (type: decimal(17,9)), -1809242.315111134 (type: decimal(17,9)) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(30,9)) sort order: + Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(30,9)) + value expressions: _col1 (type: decimal(30,9)), _col2 (type: decimal(17,9)), _col3 (type: decimal(17,9)) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: decimal(30,9)), VALUE._col0 (type: decimal(30,9)), 1809242.315111134 (type: decimal(17,9)), -1809242.315111134 (type: decimal(17,9)) + expressions: KEY.reducesinkkey0 (type: decimal(30,9)), VALUE._col0 (type: decimal(30,9)), VALUE._col1 (type: decimal(17,9)), VALUE._col2 (type: decimal(17,9)) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out index d406f2b..d69012e 100644 --- ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out +++ ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out @@ -1,4 +1,4 @@ -Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: -- HIVE-12738 -- We are checking if a MapJoin after a GroupBy will work properly. explain select * @@ -27,39 +27,39 @@ Stage-0 limit:-1 Stage-1 Reducer 2 vectorized - File Output Operator [FS_34] - Select Operator [SEL_33] (rows=302 width=10) + File Output Operator [FS_33] + Select Operator [SEL_32] (rows=302 width=10) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_22] - Select Operator [SEL_21] (rows=302 width=10) + SHUFFLE [RS_21] + Select Operator [SEL_20] (rows=302 width=10) Output:["_col0","_col1"] - Filter Operator [FIL_20] (rows=302 width=10) + Filter Operator [FIL_19] (rows=302 width=10) predicate:_col3 is null - Map Join Operator [MAPJOIN_29] (rows=605 width=10) - Conds:MAPJOIN_28._col0=RS_18._col0(Left Outer),HybridGraceHashJoin:true,Output:["_col0","_col1","_col3"] + Map Join Operator [MAPJOIN_28] (rows=605 width=10) + Conds:MAPJOIN_27._col0=RS_17._col0(Left Outer),HybridGraceHashJoin:true,Output:["_col0","_col1","_col3"] <-Map 5 [BROADCAST_EDGE] - BROADCAST [RS_18] + BROADCAST [RS_17] PartitionCols:_col0 Select Operator [SEL_12] (rows=500 width=10) Output:["_col0"] TableScan [TS_11] (rows=500 width=10) default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map Join Operator [MAPJOIN_28] (rows=550 width=10) + <-Map Join Operator [MAPJOIN_27] (rows=550 width=10) Conds:(Inner),Output:["_col0","_col1"] <-Reducer 4 [BROADCAST_EDGE] vectorized - BROADCAST [RS_15] + BROADCAST [RS_14] Select Operator [SEL_10] (rows=1 width=8) Filter Operator [FIL_9] (rows=1 width=8) predicate:(_col0 = 0) - Group By Operator [GBY_32] (rows=1 width=8) + Group By Operator [GBY_31] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 3 [SIMPLE_EDGE] SHUFFLE [RS_6] Group By Operator [GBY_5] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Select Operator [SEL_4] (rows=250 width=10) - Filter Operator [FIL_26] (rows=250 width=10) + Filter Operator [FIL_25] (rows=250 width=10) predicate:key is null TableScan [TS_2] (rows=500 width=10) default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] @@ -68,7 +68,7 @@ Stage-0 TableScan [TS_0] (rows=500 width=10) default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"] -Warning: Map Join MAPJOIN[28][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: select * from src where not key in diff --git ql/src/test/results/clientpositive/tez/vector_interval_1.q.out ql/src/test/results/clientpositive/tez/vector_interval_1.q.out index dbfa842..dbf2a5a 100644 --- ql/src/test/results/clientpositive/tez/vector_interval_1.q.out +++ ql/src/test/results/clientpositive/tez/vector_interval_1.q.out @@ -162,20 +162,20 @@ STAGE PLANS: alias: vector_interval_1 Statistics: Num rows: 2 Data size: 442 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: dt (type: date), (CAST( str1 AS INTERVAL YEAR TO MONTH) + CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), (1-2 + CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), (CAST( str1 AS INTERVAL YEAR TO MONTH) - CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), (1-2 - CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month) - outputColumnNames: _col0, _col2, _col3, _col5, _col6 + expressions: dt (type: date), 2-4 (type: interval_year_month), (CAST( str1 AS INTERVAL YEAR TO MONTH) + CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), (1-2 + CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), 0-0 (type: interval_year_month), (CAST( str1 AS INTERVAL YEAR TO MONTH) - CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), (1-2 - CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 442 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: date) sort order: + Statistics: Num rows: 2 Data size: 442 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: interval_year_month), _col3 (type: interval_year_month), _col5 (type: interval_year_month), _col6 (type: interval_year_month) + value expressions: _col1 (type: interval_year_month), _col2 (type: interval_year_month), _col3 (type: interval_year_month), _col4 (type: interval_year_month), _col5 (type: interval_year_month), _col6 (type: interval_year_month) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: date), 2-4 (type: interval_year_month), VALUE._col1 (type: interval_year_month), VALUE._col2 (type: interval_year_month), 0-0 (type: interval_year_month), VALUE._col4 (type: interval_year_month), VALUE._col5 (type: interval_year_month) + expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: interval_year_month), VALUE._col1 (type: interval_year_month), VALUE._col2 (type: interval_year_month), VALUE._col3 (type: interval_year_month), VALUE._col4 (type: interval_year_month), VALUE._col5 (type: interval_year_month) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 442 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out index 8409a01..064e319 100644 --- ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out +++ ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out @@ -510,18 +510,21 @@ STAGE PLANS: alias: interval_arithmetic_1 Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: -1-1 (type: interval_year_month) + outputColumnNames: _col1 Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator key expressions: 5-5 (type: interval_year_month) sort order: + Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: interval_year_month) Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 5-5 (type: interval_year_month), -1-1 (type: interval_year_month) + expressions: KEY.reducesinkkey0 (type: interval_year_month), VALUE._col0 (type: interval_year_month) outputColumnNames: _col0, _col1 Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Limit diff --git ql/src/test/results/clientpositive/tez/vector_join_filters.q.out ql/src/test/results/clientpositive/tez/vector_join_filters.q.out index d50e079..a010722 100644 --- ql/src/test/results/clientpositive/tez/vector_join_filters.q.out +++ ql/src/test/results/clientpositive/tez/vector_join_filters.q.out @@ -30,7 +30,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@myinput1 POSTHOOK: Lineage: myinput1.key SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: myinput1.value SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:value, type:int, comment:null), ] -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/tez/vector_join_nulls.q.out ql/src/test/results/clientpositive/tez/vector_join_nulls.q.out index 97b3242..95b35b6 100644 --- ql/src/test/results/clientpositive/tez/vector_join_nulls.q.out +++ ql/src/test/results/clientpositive/tez/vector_join_nulls.q.out @@ -30,7 +30,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@myinput1 POSTHOOK: Lineage: myinput1.key SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: myinput1.value SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:value, type:int, comment:null), ] -Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -40,7 +40,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[14][bigTable=?] in task 'Map 1' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out index 2864a48..70a402d 100644 --- ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out +++ ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out @@ -201,15 +201,15 @@ STAGE PLANS: predicate: ((l_linenumber = 1) and l_partkey is not null and l_orderkey is not null) (type: boolean) Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) - outputColumnNames: _col0, _col1, _col2 + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col0 (type: int), 1 (type: int) - 1 _col0 (type: int), 1 (type: int) + 0 _col0 (type: int), _col3 (type: int) + 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col1, _col2 input vertices: 1 Map 2 @@ -230,18 +230,18 @@ STAGE PLANS: predicate: ((l_shipmode = 'AIR') and (l_linenumber = 1) and l_orderkey is not null) (type: boolean) Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: l_orderkey (type: int) - outputColumnNames: _col0 + expressions: l_orderkey (type: int), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: int), 1 (type: int) + keys: _col0 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int), 1 (type: int) + key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ - Map-reduce partition columns: _col0 (type: int), 1 (type: int) + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE Map 3 Map Operator Tree: diff --git ql/src/test/results/clientpositive/tez/vector_null_projection.q.out ql/src/test/results/clientpositive/tez/vector_null_projection.q.out index a4ccef2..d655d78 100644 --- ql/src/test/results/clientpositive/tez/vector_null_projection.q.out +++ ql/src/test/results/clientpositive/tez/vector_null_projection.q.out @@ -120,9 +120,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator - key expressions: null (type: void) + key expressions: _col0 (type: void) sort order: + - Map-reduce partition columns: null (type: void) + Map-reduce partition columns: _col0 (type: void) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Map 4 Map Operator Tree: @@ -139,28 +139,24 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator - key expressions: null (type: void) + key expressions: _col0 (type: void) sort order: + - Map-reduce partition columns: null (type: void) + Map-reduce partition columns: _col0 (type: void) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reducer 3 Reduce Operator Tree: Group By Operator - keys: null (type: void) + keys: KEY._col0 (type: void) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: null (type: void) - outputColumnNames: _col0 + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 diff --git ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out index 946a558..e5459c8 100644 --- ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out +++ ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -65,7 +65,7 @@ POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [(alltypesorc)alltypesorc POSTHOOK: Lineage: small_alltypesorc3a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc @@ -81,13 +81,13 @@ POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [] PREHOOK: query: select * from small_alltypesorc1a PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc1a diff --git ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out index 1998344..2a8f1e0 100644 --- ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out +++ ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -54,7 +54,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default POSTHOOK: Output: default@small_alltypesorc3a -POSTHOOK: Lineage: small_alltypesorc3a.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.cbigint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc3a.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] @@ -76,12 +76,12 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default POSTHOOK: Output: default@small_alltypesorc4a -POSTHOOK: Lineage: small_alltypesorc4a.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cbigint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] diff --git ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out index f20163b..b343795 100644 --- ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out +++ ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -61,7 +61,7 @@ POSTHOOK: Lineage: small_alltypesorc3a.cdouble SIMPLE [(alltypesorc)alltypesorc. POSTHOOK: Lineage: small_alltypesorc3a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [] POSTHOOK: Lineage: small_alltypesorc3a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] @@ -81,9 +81,9 @@ POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] diff --git ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out index 90a9efb..8ae89e8 100644 --- ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out +++ ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2b.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2b.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2b.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2b.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2b.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -65,7 +65,7 @@ POSTHOOK: Lineage: small_alltypesorc3b.cstring1 SIMPLE [(alltypesorc)alltypesorc POSTHOOK: Lineage: small_alltypesorc3b.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3b.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3b.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3b.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3b.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc @@ -81,13 +81,13 @@ POSTHOOK: Lineage: small_alltypesorc4b.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4b.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4b.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4b.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4b.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4b.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4b.ctinyint SIMPLE [] PREHOOK: query: select * from small_alltypesorc1b PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc1b diff --git ql/src/test/results/clientpositive/tez/vector_outer_join5.q.out ql/src/test/results/clientpositive/tez/vector_outer_join5.q.out index c1c251f..cb6fda0 100644 --- ql/src/test/results/clientpositive/tez/vector_outer_join5.q.out +++ ql/src/test/results/clientpositive/tez/vector_outer_join5.q.out @@ -569,7 +569,7 @@ STAGE PLANS: Left Outer Join0 to 1 keys: 0 UDFToLong(_col1) (type: bigint) - 1 (_col0 pmod UDFToLong(8)) (type: bigint) + 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 input vertices: 1 Map 4 @@ -591,9 +591,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (_col0 pmod UDFToLong(8)) (type: bigint) + key expressions: (_col0 pmod 8) (type: bigint) sort order: + - Map-reduce partition columns: (_col0 pmod UDFToLong(8)) (type: bigint) + Map-reduce partition columns: (_col0 pmod 8) (type: bigint) Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 5 @@ -1239,7 +1239,7 @@ STAGE PLANS: Left Outer Join0 to 1 keys: 0 UDFToLong(_col1) (type: bigint) - 1 (_col0 pmod UDFToLong(8)) (type: bigint) + 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 input vertices: 1 Map 4 @@ -1261,9 +1261,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (_col0 pmod UDFToLong(8)) (type: bigint) + key expressions: (_col0 pmod 8) (type: bigint) sort order: + - Map-reduce partition columns: (_col0 pmod UDFToLong(8)) (type: bigint) + Map-reduce partition columns: (_col0 pmod 8) (type: bigint) Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Map 5 diff --git ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out index 70f8d1b..a686ace 100644 --- ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out +++ ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out @@ -2356,7 +2356,7 @@ STAGE PLANS: alias: alltypesorc Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(ctimestamp1) <> 0.0) and (((-257 <> UDFToInteger(ctinyint)) and cstring1 regexp '.*ss' and (-3.0 < UDFToDouble(ctimestamp1)) and cboolean2 is not null) or (UDFToDouble(ctimestamp2) = -5.0) or ((UDFToDouble(ctimestamp1) < 0.0) and (cstring2 like '%b%')) or (cdouble = UDFToDouble(cint)) or (cboolean1 is null and (cfloat < UDFToFloat(cint))))) (type: boolean) + predicate: ((UDFToDouble(ctimestamp1) <> 0.0) and (((-257 <> UDFToInteger(ctinyint)) and cboolean2 is not null and cstring1 regexp '.*ss' and (-3.0 < UDFToDouble(ctimestamp1))) or (UDFToDouble(ctimestamp2) = -5.0) or ((UDFToDouble(ctimestamp1) < 0.0) and (cstring2 like '%b%')) or (cdouble = UDFToDouble(cint)) or (cboolean1 is null and (cfloat < UDFToFloat(cint))))) (type: boolean) Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cstring1 (type: string), ctimestamp1 (type: timestamp), cint (type: int), csmallint (type: smallint), ctinyint (type: tinyint), cfloat (type: float), cdouble (type: double) @@ -2690,7 +2690,7 @@ STAGE PLANS: alias: alltypesorc Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((cdouble < UDFToDouble(csmallint)) and (cboolean2 = cboolean1) and (UDFToDouble(cbigint) <= -863.257)) or ((cint >= -257) and (cboolean1 >= 1) and cstring1 is not null) or cstring2 regexp 'b' or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null)) and cboolean1 is not null) (type: boolean) + predicate: ((((cdouble < UDFToDouble(csmallint)) and (cboolean2 = cboolean1) and (UDFToDouble(cbigint) <= -863.257)) or ((cint >= -257) and cstring1 is not null and (cboolean1 >= 1)) or cstring2 regexp 'b' or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null)) and cboolean1 is not null) (type: boolean) Statistics: Num rows: 10239 Data size: 2201421 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cbigint (type: bigint), cint (type: int), cdouble (type: double), ctinyint (type: tinyint), csmallint (type: smallint) diff --git ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out index a790b97..7ca9604 100644 --- ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out @@ -1348,12 +1348,12 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1375,7 +1375,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1455,9 +1455,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + key expressions: (UDFToDouble(_col0) * 2.0) (type: double) sort order: + - Map-reduce partition columns: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + Map-reduce partition columns: (UDFToDouble(_col0) * 2.0) (type: double) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: @@ -1499,7 +1499,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1598,9 +1598,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Reducer 2 @@ -1610,7 +1610,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1690,9 +1690,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + key expressions: (UDFToDouble(_col0) * 2.0) (type: double) sort order: + - Map-reduce partition columns: (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + Map-reduce partition columns: (UDFToDouble(_col0) * 2.0) (type: double) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: @@ -1719,7 +1719,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1813,9 +1813,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + key expressions: UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) sort order: + - Map-reduce partition columns: UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + Map-reduce partition columns: UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: @@ -1857,7 +1857,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 UDFToString((UDFToDouble(_col0) * UDFToDouble(2))) (type: string) + 0 UDFToString((UDFToDouble(_col0) * 2.0)) (type: string) 1 UDFToString(_col0) (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -1923,6 +1923,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -1947,42 +1948,40 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + sort order: + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: '2008-04-08' (type: string) - mode: hash - outputColumnNames: _col0 + Select Operator Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '2008-04-08' (type: string) - sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Merge Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) - 1 '2008-04-08' (type: string) - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + 0 + 1 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -2011,32 +2010,15 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Group By Operator - keys: '2008-04-08' (type: string) + keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: '2008-04-08' (type: string) - sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: _col0 + sort order: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Dynamic Partitioning Event Operator - Target column: ds (string) - Target Input: srcpart - Partition key expr: ds - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -2044,21 +2026,18 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -2074,7 +2053,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- non-equi join EXPLAIN select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY @@ -2171,7 +2150,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart, srcpart_date_hour where (srcpart_date_hour.`date` = '2008-04-08' and srcpart_date_hour.hour = 11) and (srcpart.ds = srcpart_date_hour.ds or srcpart.hr = srcpart_date_hour.hr) PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -4247,7 +4226,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + 1 UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) input vertices: 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -4275,12 +4254,12 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + key expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) sort order: + - Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + Map-reduce partition columns: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(UDFToInteger((_col0 / UDFToDouble(2)))) (type: double) + expressions: UDFToDouble(UDFToInteger((_col0 / 2.0))) (type: double) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -4367,7 +4346,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 (UDFToDouble(_col0) * UDFToDouble(2)) (type: double) + 0 (UDFToDouble(_col0) * 2.0) (type: double) 1 _col0 (type: double) input vertices: 1 Map 3 @@ -4470,6 +4449,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 +Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -4485,57 +4465,68 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 4 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan alias: srcpart - filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + filterExpr: (ds = '2008-04-08') (type: boolean) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ds (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 '2008-04-08' (type: string) - input vertices: - 1 Reducer 4 - Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - HybridGraceHashJoin: true - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 3 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Map 2 Map Operator Tree: TableScan alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: '2008-04-08' (type: string) - mode: hash - outputColumnNames: _col0 + Select Operator Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '2008-04-08' (type: string) - sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reducer 2 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + input vertices: + 0 Map 1 + Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 4 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -4550,36 +4541,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: '2008-04-08' (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '2008-04-08' (type: string) - sort order: + - Map-reduce partition columns: '2008-04-08' (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Dynamic Partitioning Event Operator - Target column: ds (string) - Target Input: srcpart - Partition key expr: ds - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -4587,21 +4548,18 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' diff --git ql/src/test/results/clientpositive/udf_folder_constants.q.out ql/src/test/results/clientpositive/udf_folder_constants.q.out index 3e765d9..6ee520f 100644 --- ql/src/test/results/clientpositive/udf_folder_constants.q.out +++ ql/src/test/results/clientpositive/udf_folder_constants.q.out @@ -63,12 +63,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: month (type: int) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: int) + key expressions: _col0 (type: int) sort order: + - Map-reduce partition columns: _col1 (type: int) + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE TableScan alias: b @@ -90,7 +90,7 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col1 (type: int) + 0 _col0 (type: int) 1 _col0 (type: int) Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE Select Operator diff --git ql/src/test/results/clientpositive/udf_unix_timestamp.q.out ql/src/test/results/clientpositive/udf_unix_timestamp.q.out index 1aa9727..a1543d9 100644 --- ql/src/test/results/clientpositive/udf_unix_timestamp.q.out +++ ql/src/test/results/clientpositive/udf_unix_timestamp.q.out @@ -89,7 +89,7 @@ POSTHOOK: Input: default@oneline POSTHOOK: Output: database:default POSTHOOK: Output: default@foo POSTHOOK: Lineage: foo.a SIMPLE [] -POSTHOOK: Lineage: foo.b EXPRESSION [] +POSTHOOK: Lineage: foo.b SIMPLE [] PREHOOK: query: drop table foo PREHOOK: type: DROPTABLE PREHOOK: Input: default@foo diff --git ql/src/test/results/clientpositive/union_fast_stats.q.out ql/src/test/results/clientpositive/union_fast_stats.q.out index f0879af..c3e9664 100644 --- ql/src/test/results/clientpositive/union_fast_stats.q.out +++ ql/src/test/results/clientpositive/union_fast_stats.q.out @@ -55,7 +55,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -83,7 +83,7 @@ POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [(alltypesorc)alltypesorc POSTHOOK: Lineage: small_alltypesorc3a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc @@ -99,13 +99,13 @@ POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from (select * from (select * from small_alltypesorc1a) sq1 union all @@ -388,7 +388,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -416,7 +416,7 @@ POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [(alltypesorc)alltypesorc POSTHOOK: Lineage: small_alltypesorc3a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc @@ -432,13 +432,13 @@ POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from (select * from (select * from small_alltypesorc1a) sq1 union all diff --git ql/src/test/results/clientpositive/union_offcbo.q.out ql/src/test/results/clientpositive/union_offcbo.q.out index 7eff464..b8a6770 100644 --- ql/src/test/results/clientpositive/union_offcbo.q.out +++ ql/src/test/results/clientpositive/union_offcbo.q.out @@ -629,10 +629,10 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((CASE WHEN ((_col7 is not null and _col0 is null and (_col3 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col7 is not null and _col0 is null and (_col3 <= '2016-02-05'))) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END <> 'RET') and ((NVL(_col0,-1) <> NVL(_col7,-1)) or (NVL(_col1,-1) <> NVL(_col8,-1)))) (type: boolean) + predicate: ((CASE WHEN ((_col0 is null and (_col3 >= '2016-02-05') and _col7 is not null)) THEN ('DEL') WHEN ((_col0 is null and (_col3 <= '2016-02-05') and _col7 is not null)) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END <> 'RET') and (not ((NVL(_col0,-1) = NVL(_col7,-1)) and (NVL(_col1,-1) = NVL(_col8,-1))))) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col2 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col4 (type: string), _col7 (type: string), _col8 (type: string), CASE WHEN ((_col7 is not null and _col0 is null and (_col3 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col7 is not null and _col0 is null and (_col3 <= '2016-02-05'))) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END (type: string) + expressions: _col2 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col4 (type: string), _col7 (type: string), _col8 (type: string), CASE WHEN ((_col0 is null and (_col3 >= '2016-02-05') and _col7 is not null)) THEN ('DEL') WHEN ((_col0 is null and (_col3 <= '2016-02-05') and _col7 is not null)) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator @@ -684,15 +684,15 @@ STAGE PLANS: predicate: ((ts1 = '2015-11-20') and reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: id1 (type: bigint), '2015-11-20' (type: string), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + expressions: id1 (type: bigint), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col4 (type: string) + key expressions: _col3 (type: string) sort order: + - Map-reduce partition columns: _col4 (type: string) + Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string), _col3 (type: bigint), _col5 (type: string) + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string) TableScan alias: ttest2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -714,15 +714,15 @@ STAGE PLANS: condition map: Left Outer Join0 to 1 keys: - 0 _col4 (type: string) + 0 _col3 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((CASE WHEN ((_col7 is not null and _col4 is null and (_col6 <= _col1))) THEN ('DEL') WHEN (((_col7 is null and _col4 is not null) or ((_col7 = _col4) and (_col8 <> _col5)))) THEN ('INS') ELSE ('NA') END <> 'RET') and ((NVL(_col4,-1) <> NVL(_col7,-1)) or (NVL(_col5,-1) <> NVL(_col8,-1)))) (type: boolean) + predicate: ((CASE WHEN ((_col3 is null and (_col5 <= '2015-11-20') and _col6 is not null)) THEN ('DEL') WHEN (((_col6 is null and _col3 is not null) or ((_col6 = _col3) and (_col7 <> _col4)))) THEN ('INS') ELSE ('NA') END <> 'RET') and (not ((NVL(_col3,-1) = NVL(_col6,-1)) and (NVL(_col4,-1) = NVL(_col7,-1))))) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col0 (type: bigint), _col2 (type: string), _col3 (type: bigint), '2099-12-31' (type: string), _col4 (type: string), _col5 (type: string), CASE WHEN ((_col7 is not null and _col4 is null and (_col6 <= _col1))) THEN ('DEL') WHEN (((_col7 is null and _col4 is not null) or ((_col7 = _col4) and (_col8 <> _col5)))) THEN ('INS') ELSE ('NA') END (type: string) + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), '2099-12-31' (type: string), _col3 (type: string), _col4 (type: string), CASE WHEN ((_col3 is null and (_col5 <= '2015-11-20') and _col6 is not null)) THEN ('DEL') WHEN (((_col6 is null and _col3 is not null) or ((_col6 = _col3) and (_col7 <> _col4)))) THEN ('INS') ELSE ('NA') END (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator @@ -1652,10 +1652,10 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((CASE WHEN ((_col7 is not null and _col0 is null and (_col3 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col7 is not null and _col0 is null and (_col3 <= '2016-02-05'))) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END <> 'RET') and ((NVL(_col0,-1) <> NVL(_col7,-1)) or (NVL(_col1,-1) <> NVL(_col8,-1)))) (type: boolean) + predicate: ((CASE WHEN ((_col0 is null and (_col3 >= '2016-02-05') and _col7 is not null)) THEN ('DEL') WHEN ((_col0 is null and (_col3 <= '2016-02-05') and _col7 is not null)) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END <> 'RET') and (not ((NVL(_col0,-1) = NVL(_col7,-1)) and (NVL(_col1,-1) = NVL(_col8,-1))))) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col2 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col4 (type: string), _col7 (type: string), _col8 (type: string), CASE WHEN ((_col7 is not null and _col0 is null and (_col3 >= '2016-02-05'))) THEN ('DEL') WHEN ((_col7 is not null and _col0 is null and (_col3 <= '2016-02-05'))) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END (type: string) + expressions: _col2 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col4 (type: string), _col7 (type: string), _col8 (type: string), CASE WHEN ((_col0 is null and (_col3 >= '2016-02-05') and _col7 is not null)) THEN ('DEL') WHEN ((_col0 is null and (_col3 <= '2016-02-05') and _col7 is not null)) THEN ('RET') WHEN (((_col7 = _col0) and (_col8 <> _col1))) THEN ('A_INS') ELSE ('NA') END (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator @@ -1707,15 +1707,15 @@ STAGE PLANS: predicate: ((ts1 = '2015-11-20') and reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) is not null) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: id1 (type: bigint), '2015-11-20' (type: string), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + expressions: id1 (type: bigint), sts (type: string), at1 (type: bigint), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(id1)) (type: string), reflect('org.apache.commons.codec.digest.DigestUtils','sha256Hex',concat(at1)) (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col4 (type: string) + key expressions: _col3 (type: string) sort order: + - Map-reduce partition columns: _col4 (type: string) + Map-reduce partition columns: _col3 (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string), _col3 (type: bigint), _col5 (type: string) + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), _col4 (type: string) TableScan alias: ttest2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -1737,15 +1737,15 @@ STAGE PLANS: condition map: Left Outer Join0 to 1 keys: - 0 _col4 (type: string) + 0 _col3 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator - predicate: ((CASE WHEN ((_col7 is not null and _col4 is null and (_col6 <= _col1))) THEN ('DEL') WHEN (((_col7 is null and _col4 is not null) or ((_col7 = _col4) and (_col8 <> _col5)))) THEN ('INS') ELSE ('NA') END <> 'RET') and ((NVL(_col4,-1) <> NVL(_col7,-1)) or (NVL(_col5,-1) <> NVL(_col8,-1)))) (type: boolean) + predicate: ((CASE WHEN ((_col3 is null and (_col5 <= '2015-11-20') and _col6 is not null)) THEN ('DEL') WHEN (((_col6 is null and _col3 is not null) or ((_col6 = _col3) and (_col7 <> _col4)))) THEN ('INS') ELSE ('NA') END <> 'RET') and (not ((NVL(_col3,-1) = NVL(_col6,-1)) and (NVL(_col4,-1) = NVL(_col7,-1))))) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col0 (type: bigint), _col2 (type: string), _col3 (type: bigint), '2099-12-31' (type: string), _col4 (type: string), _col5 (type: string), CASE WHEN ((_col7 is not null and _col4 is null and (_col6 <= _col1))) THEN ('DEL') WHEN (((_col7 is null and _col4 is not null) or ((_col7 = _col4) and (_col8 <> _col5)))) THEN ('INS') ELSE ('NA') END (type: string) + expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: bigint), '2099-12-31' (type: string), _col3 (type: string), _col4 (type: string), CASE WHEN ((_col3 is null and (_col5 <= '2015-11-20') and _col6 is not null)) THEN ('DEL') WHEN (((_col6 is null and _col3 is not null) or ((_col6 = _col3) and (_col7 <> _col4)))) THEN ('INS') ELSE ('NA') END (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/union_remove_12.q.out ql/src/test/results/clientpositive/union_remove_12.q.out index 2b42538..5f73c9a 100644 --- ql/src/test/results/clientpositive/union_remove_12.q.out +++ ql/src/test/results/clientpositive/union_remove_12.q.out @@ -87,7 +87,7 @@ STAGE PLANS: alias: inputtbl1 Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), UDFToLong('1') (type: bigint) + expressions: key (type: string), 1 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/union_remove_14.q.out ql/src/test/results/clientpositive/union_remove_14.q.out index a754dd4..52dc7c5 100644 --- ql/src/test/results/clientpositive/union_remove_14.q.out +++ ql/src/test/results/clientpositive/union_remove_14.q.out @@ -89,7 +89,7 @@ STAGE PLANS: alias: inputtbl1 Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), UDFToLong('1') (type: bigint) + expressions: key (type: string), 1 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/union_view.q.out ql/src/test/results/clientpositive/union_view.q.out index 530739e..90e240a 100644 --- ql/src/test/results/clientpositive/union_view.q.out +++ ql/src/test/results/clientpositive/union_view.q.out @@ -706,14 +706,14 @@ STAGE PLANS: Union Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col1, _col2 + expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) sort order: + Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) TableScan alias: src_union_2 filterExpr: ((key = 86) and ds is not null) (type: boolean) @@ -728,14 +728,14 @@ STAGE PLANS: Union Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col1, _col2 + expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) sort order: + Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) TableScan alias: src_union_3 filterExpr: ((key = 86) and ds is not null) (type: boolean) @@ -750,17 +750,17 @@ STAGE PLANS: Union Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col1, _col2 + expressions: 86 (type: int), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) sort order: + Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: 86 (type: int), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out index ba3a0b8..5e166e6 100644 --- ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out +++ ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out @@ -386,20 +386,18 @@ STAGE PLANS: predicate: (f1 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 1 (type: int) - outputColumnNames: _col0 + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe TableScan alias: union_all_bug_test_2 Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE @@ -407,20 +405,18 @@ STAGE PLANS: predicate: (f1 = 1) (type: boolean) Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 1 (type: int) - outputColumnNames: _col0 + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/vector_between_columns.q.out ql/src/test/results/clientpositive/vector_between_columns.q.out index 5faa79b..5143074 100644 --- ql/src/test/results/clientpositive/vector_between_columns.q.out +++ ql/src/test/results/clientpositive/vector_between_columns.q.out @@ -70,7 +70,7 @@ POSTHOOK: Output: default@TINT POSTHOOK: Lineage: tint.cint SIMPLE [(tint_txt)tint_txt.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: tint.rnum SIMPLE [(tint_txt)tint_txt.FieldSchema(name:rnum, type:int, comment:null), ] tint_txt.rnum tint_txt.cint -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: explain select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where tint.cint between tsint.csint and tsint.csint PREHOOK: type: QUERY @@ -145,7 +145,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[10][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where tint.cint between tsint.csint and tsint.csint PREHOOK: type: QUERY PREHOOK: Input: default@tint diff --git ql/src/test/results/clientpositive/vector_binary_join_groupby.q.out ql/src/test/results/clientpositive/vector_binary_join_groupby.q.out index 2169db9..d9c027a 100644 --- ql/src/test/results/clientpositive/vector_binary_join_groupby.q.out +++ ql/src/test/results/clientpositive/vector_binary_join_groupby.q.out @@ -190,7 +190,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(*)) FROM hundredorc t1 JOIN hundredorc t2 ON t2.bin = t2.bin PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_coalesce.q.out ql/src/test/results/clientpositive/vector_coalesce.q.out index e126dcb..21a9122 100644 --- ql/src/test/results/clientpositive/vector_coalesce.q.out +++ ql/src/test/results/clientpositive/vector_coalesce.q.out @@ -40,7 +40,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: null (type: double), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: smallint), KEY.reducesinkkey5 (type: string) + expressions: KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: smallint), KEY.reducesinkkey5 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Limit @@ -124,7 +124,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: null (type: tinyint), KEY.reducesinkkey1 (type: double), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: double) + expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: double) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Limit @@ -206,7 +206,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: null (type: float), null (type: bigint), 0.0 (type: float) + expressions: KEY.reducesinkkey0 (type: float), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey2 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE Limit @@ -372,7 +372,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: null (type: float), null (type: bigint), null (type: float) + expressions: KEY.reducesinkkey0 (type: float), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey0 (type: float) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE Limit @@ -443,7 +443,7 @@ STAGE PLANS: predicate: cbigint is null (type: boolean) Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: null (type: bigint), ctinyint (type: tinyint), COALESCE(null,ctinyint) (type: bigint) + expressions: null (type: bigint), ctinyint (type: tinyint), COALESCE(null,ctinyint) (type: tinyint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE Limit diff --git ql/src/test/results/clientpositive/vector_date_1.q.out ql/src/test/results/clientpositive/vector_date_1.q.out index da608bf..a394e0f 100644 --- ql/src/test/results/clientpositive/vector_date_1.q.out +++ ql/src/test/results/clientpositive/vector_date_1.q.out @@ -617,7 +617,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 2001-01-01 (type: date), VALUE._col0 (type: date) + expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: date) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 74 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/vector_decimal_2.q.out ql/src/test/results/clientpositive/vector_decimal_2.q.out index 5e5b36c..892a1b3 100644 --- ql/src/test/results/clientpositive/vector_decimal_2.q.out +++ ql/src/test/results/clientpositive/vector_decimal_2.q.out @@ -914,15 +914,18 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3.14 (type: decimal(4,2)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3.14 (type: decimal(3,2)) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(4,2)) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3.14 (type: decimal(4,2)) + expressions: VALUE._col0 (type: decimal(4,2)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -966,15 +969,18 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3.14 (type: decimal(4,2)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3.14 (type: decimal(3,2)) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(4,2)) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3.14 (type: decimal(4,2)) + expressions: VALUE._col0 (type: decimal(4,2)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1018,15 +1024,18 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 1355944339.1234567 (type: decimal(30,8)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 1355944339.1234567 (type: decimal(17,7)) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(30,8)) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 1355944339.1234567 (type: decimal(30,8)) + expressions: VALUE._col0 (type: decimal(30,8)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1070,15 +1079,18 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 1 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 1 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 1 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1113,15 +1125,18 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 1 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 1 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 1 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1165,15 +1180,18 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1217,15 +1235,18 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1269,15 +1290,18 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1321,15 +1345,18 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 3 (type: decimal(10,0)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 3 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,0)) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 3 (type: decimal(10,0)) + expressions: VALUE._col0 (type: decimal(10,0)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1373,15 +1400,18 @@ STAGE PLANS: alias: decimal_2 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: 1 (type: decimal(20,19)) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: 1 (type: int) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(20,19)) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 1 (type: decimal(20,19)) + expressions: VALUE._col0 (type: decimal(20,19)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -1433,7 +1463,7 @@ STAGE PLANS: Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 0.99999999999999999999 (type: decimal(20,20)) + expressions: KEY.reducesinkkey0 (type: decimal(20,20)) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out index d367b1e..55262de 100644 --- ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out +++ ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out @@ -1,4 +1,4 @@ -Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[32][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: -- HIVE-12738 -- We are checking if a MapJoin after a GroupBy will work properly. explain select * @@ -148,7 +148,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[33][bigTable=?] in task 'Stage-3:MAPRED' is a cross product +Warning: Map Join MAPJOIN[32][bigTable=?] in task 'Stage-3:MAPRED' is a cross product PREHOOK: query: select * from src where not key in diff --git ql/src/test/results/clientpositive/vector_interval_1.q.out ql/src/test/results/clientpositive/vector_interval_1.q.out index 6845628..379747c 100644 --- ql/src/test/results/clientpositive/vector_interval_1.q.out +++ ql/src/test/results/clientpositive/vector_interval_1.q.out @@ -148,18 +148,18 @@ STAGE PLANS: alias: vector_interval_1 Statistics: Num rows: 2 Data size: 442 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: dt (type: date), (CAST( str1 AS INTERVAL YEAR TO MONTH) + CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), (1-2 + CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), (CAST( str1 AS INTERVAL YEAR TO MONTH) - CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), (1-2 - CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month) - outputColumnNames: _col0, _col2, _col3, _col5, _col6 + expressions: dt (type: date), 2-4 (type: interval_year_month), (CAST( str1 AS INTERVAL YEAR TO MONTH) + CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), (1-2 + CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), 0-0 (type: interval_year_month), (CAST( str1 AS INTERVAL YEAR TO MONTH) - CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month), (1-2 - CAST( str1 AS INTERVAL YEAR TO MONTH)) (type: interval_year_month) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 442 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: date) sort order: + Statistics: Num rows: 2 Data size: 442 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: interval_year_month), _col3 (type: interval_year_month), _col5 (type: interval_year_month), _col6 (type: interval_year_month) + value expressions: _col1 (type: interval_year_month), _col2 (type: interval_year_month), _col3 (type: interval_year_month), _col4 (type: interval_year_month), _col5 (type: interval_year_month), _col6 (type: interval_year_month) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: date), 2-4 (type: interval_year_month), VALUE._col1 (type: interval_year_month), VALUE._col2 (type: interval_year_month), 0-0 (type: interval_year_month), VALUE._col4 (type: interval_year_month), VALUE._col5 (type: interval_year_month) + expressions: KEY.reducesinkkey0 (type: date), VALUE._col0 (type: interval_year_month), VALUE._col1 (type: interval_year_month), VALUE._col2 (type: interval_year_month), VALUE._col3 (type: interval_year_month), VALUE._col4 (type: interval_year_month), VALUE._col5 (type: interval_year_month) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 2 Data size: 442 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out index cd8111d..bad7e4a 100644 --- ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out +++ ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out @@ -480,16 +480,19 @@ STAGE PLANS: alias: interval_arithmetic_1 Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: -1-1 (type: interval_year_month) + outputColumnNames: _col1 Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator key expressions: 5-5 (type: interval_year_month) sort order: + Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: interval_year_month) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: 5-5 (type: interval_year_month), -1-1 (type: interval_year_month) + expressions: KEY.reducesinkkey0 (type: interval_year_month), VALUE._col0 (type: interval_year_month) outputColumnNames: _col0, _col1 Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Limit diff --git ql/src/test/results/clientpositive/vector_join_filters.q.out ql/src/test/results/clientpositive/vector_join_filters.q.out index 999fee7..30003d7 100644 --- ql/src/test/results/clientpositive/vector_join_filters.q.out +++ ql/src/test/results/clientpositive/vector_join_filters.q.out @@ -30,7 +30,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@myinput1 POSTHOOK: Lineage: myinput1.key SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: myinput1.value SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:value, type:int, comment:null), ] -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/vector_join_nulls.q.out ql/src/test/results/clientpositive/vector_join_nulls.q.out index 9011a1f..fdffbb6 100644 --- ql/src/test/results/clientpositive/vector_join_nulls.q.out +++ ql/src/test/results/clientpositive/vector_join_nulls.q.out @@ -30,7 +30,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@myinput1 POSTHOOK: Lineage: myinput1.key SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: myinput1.value SIMPLE [(myinput1_txt)myinput1_txt.FieldSchema(name:value, type:int, comment:null), ] -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 @@ -40,7 +40,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@myinput1 #### A masked pattern was here #### 13630578 -Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b PREHOOK: type: QUERY PREHOOK: Input: default@myinput1 diff --git ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out index 20f79c1..b37d1fc 100644 --- ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out +++ ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out @@ -469,18 +469,18 @@ STAGE PLANS: predicate: ((l_shipmode = 'AIR') and (l_linenumber = 1) and l_orderkey is not null) (type: boolean) Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: l_orderkey (type: int) - outputColumnNames: _col0 + expressions: l_orderkey (type: int), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: int), 1 (type: int) + keys: _col0 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col0 (type: int), 1 (type: int) - 1 _col0 (type: int), 1 (type: int) + 0 _col0 (type: int), _col3 (type: int) + 1 _col0 (type: int), _col1 (type: int) Stage: Stage-8 Map Reduce @@ -492,15 +492,15 @@ STAGE PLANS: predicate: ((l_linenumber = 1) and l_partkey is not null and l_orderkey is not null) (type: boolean) Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) - outputColumnNames: _col0, _col1, _col2 + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col0 (type: int), 1 (type: int) - 1 _col0 (type: int), 1 (type: int) + 0 _col0 (type: int), _col3 (type: int) + 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col1, _col2 Statistics: Num rows: 55 Data size: 6598 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/vector_null_projection.q.out ql/src/test/results/clientpositive/vector_null_projection.q.out index 7517cc2..6daef59 100644 --- ql/src/test/results/clientpositive/vector_null_projection.q.out +++ ql/src/test/results/clientpositive/vector_null_projection.q.out @@ -111,9 +111,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator - key expressions: null (type: void) + key expressions: _col0 (type: void) sort order: + - Map-reduce partition columns: null (type: void) + Map-reduce partition columns: _col0 (type: void) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE TableScan alias: b @@ -130,27 +130,23 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator - key expressions: null (type: void) + key expressions: _col0 (type: void) sort order: + - Map-reduce partition columns: null (type: void) + Map-reduce partition columns: _col0 (type: void) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Operator Tree: Group By Operator - keys: null (type: void) + keys: KEY._col0 (type: void) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - expressions: null (type: void) - outputColumnNames: _col0 + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/vector_number_compare_projection.q.out ql/src/test/results/clientpositive/vector_number_compare_projection.q.out index 85f7cc8..4f1a26c 100644 --- ql/src/test/results/clientpositive/vector_number_compare_projection.q.out +++ ql/src/test/results/clientpositive/vector_number_compare_projection.q.out @@ -547,15 +547,15 @@ POSTHOOK: Input: default@vectortab2k POSTHOOK: Output: database:default POSTHOOK: Output: default@scratch_null POSTHOOK: Lineage: scratch_null.b SIMPLE [(vectortab2k)vectortab2k.FieldSchema(name:b, type:bigint, comment:null), ] -POSTHOOK: Lineage: scratch_null.b_null EXPRESSION [] +POSTHOOK: Lineage: scratch_null.b_null SIMPLE [] POSTHOOK: Lineage: scratch_null.bo SIMPLE [(vectortab2k)vectortab2k.FieldSchema(name:bo, type:boolean, comment:null), ] -POSTHOOK: Lineage: scratch_null.bo_null EXPRESSION [] +POSTHOOK: Lineage: scratch_null.bo_null SIMPLE [] POSTHOOK: Lineage: scratch_null.i SIMPLE [(vectortab2k)vectortab2k.FieldSchema(name:i, type:int, comment:null), ] -POSTHOOK: Lineage: scratch_null.i_null EXPRESSION [] +POSTHOOK: Lineage: scratch_null.i_null SIMPLE [] POSTHOOK: Lineage: scratch_null.si SIMPLE [(vectortab2k)vectortab2k.FieldSchema(name:si, type:smallint, comment:null), ] -POSTHOOK: Lineage: scratch_null.si_null EXPRESSION [] +POSTHOOK: Lineage: scratch_null.si_null SIMPLE [] POSTHOOK: Lineage: scratch_null.t SIMPLE [(vectortab2k)vectortab2k.FieldSchema(name:t, type:tinyint, comment:null), ] -POSTHOOK: Lineage: scratch_null.t_null EXPRESSION [] +POSTHOOK: Lineage: scratch_null.t_null SIMPLE [] t si i b bo t_null si_null i_null b_null bo_null PREHOOK: query: -- The nulled columns ought to create repeated null VectorizedRowBatch for those columns. CREATE TABLE vectortab2k_orc_null STORED AS ORC AS SELECT * FROM scratch_null diff --git ql/src/test/results/clientpositive/vector_outer_join1.q.out ql/src/test/results/clientpositive/vector_outer_join1.q.out index 93ab473..1e8ce11 100644 --- ql/src/test/results/clientpositive/vector_outer_join1.q.out +++ ql/src/test/results/clientpositive/vector_outer_join1.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -65,7 +65,7 @@ POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [(alltypesorc)alltypesorc POSTHOOK: Lineage: small_alltypesorc3a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc @@ -81,13 +81,13 @@ POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.ctinyint SIMPLE [] PREHOOK: query: select * from small_alltypesorc1a PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc1a diff --git ql/src/test/results/clientpositive/vector_outer_join2.q.out ql/src/test/results/clientpositive/vector_outer_join2.q.out index c8001e0..47beffd 100644 --- ql/src/test/results/clientpositive/vector_outer_join2.q.out +++ ql/src/test/results/clientpositive/vector_outer_join2.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -54,7 +54,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default POSTHOOK: Output: default@small_alltypesorc3a -POSTHOOK: Lineage: small_alltypesorc3a.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.cbigint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc3a.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] @@ -76,12 +76,12 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default POSTHOOK: Output: default@small_alltypesorc4a -POSTHOOK: Lineage: small_alltypesorc4a.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cbigint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] diff --git ql/src/test/results/clientpositive/vector_outer_join3.q.out ql/src/test/results/clientpositive/vector_outer_join3.q.out index 4f1a98d..a0c4709 100644 --- ql/src/test/results/clientpositive/vector_outer_join3.q.out +++ ql/src/test/results/clientpositive/vector_outer_join3.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -61,7 +61,7 @@ POSTHOOK: Lineage: small_alltypesorc3a.cdouble SIMPLE [(alltypesorc)alltypesorc. POSTHOOK: Lineage: small_alltypesorc3a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3a.cstring1 SIMPLE [] POSTHOOK: Lineage: small_alltypesorc3a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] @@ -81,9 +81,9 @@ POSTHOOK: Lineage: small_alltypesorc4a.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4a.cstring1 SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] diff --git ql/src/test/results/clientpositive/vector_outer_join4.q.out ql/src/test/results/clientpositive/vector_outer_join4.q.out index a32f585..eaf419c 100644 --- ql/src/test/results/clientpositive/vector_outer_join4.q.out +++ ql/src/test/results/clientpositive/vector_outer_join4.q.out @@ -37,7 +37,7 @@ POSTHOOK: Lineage: small_alltypesorc2b.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc2b.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc2b.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc2b.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc2b.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc2b.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] @@ -65,7 +65,7 @@ POSTHOOK: Lineage: small_alltypesorc3b.cstring1 SIMPLE [(alltypesorc)alltypesorc POSTHOOK: Lineage: small_alltypesorc3b.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3b.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc3b.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc3b.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc3b.ctinyint SIMPLE [] PREHOOK: query: create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc @@ -81,13 +81,13 @@ POSTHOOK: Lineage: small_alltypesorc4b.cboolean1 SIMPLE [(alltypesorc)alltypesor POSTHOOK: Lineage: small_alltypesorc4b.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4b.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4b.cint SIMPLE [] POSTHOOK: Lineage: small_alltypesorc4b.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: small_alltypesorc4b.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc4b.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +POSTHOOK: Lineage: small_alltypesorc4b.ctinyint SIMPLE [] PREHOOK: query: select * from small_alltypesorc1b PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc1b diff --git ql/src/test/results/clientpositive/vector_outer_join5.q.out ql/src/test/results/clientpositive/vector_outer_join5.q.out index 1b09fda..4c2d0c0 100644 --- ql/src/test/results/clientpositive/vector_outer_join5.q.out +++ ql/src/test/results/clientpositive/vector_outer_join5.q.out @@ -561,7 +561,7 @@ STAGE PLANS: HashTable Sink Operator keys: 0 UDFToLong(_col1) (type: bigint) - 1 (_col0 pmod UDFToLong(8)) (type: bigint) + 1 (_col0 pmod 8) (type: bigint) $hdt$_2:s TableScan alias: s @@ -590,7 +590,7 @@ STAGE PLANS: Left Outer Join0 to 1 keys: 0 UDFToLong(_col1) (type: bigint) - 1 (_col0 pmod UDFToLong(8)) (type: bigint) + 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE Map Join Operator @@ -1212,7 +1212,7 @@ STAGE PLANS: HashTable Sink Operator keys: 0 UDFToLong(_col1) (type: bigint) - 1 (_col0 pmod UDFToLong(8)) (type: bigint) + 1 (_col0 pmod 8) (type: bigint) $hdt$_2:s TableScan alias: s @@ -1241,7 +1241,7 @@ STAGE PLANS: Left Outer Join0 to 1 keys: 0 UDFToLong(_col1) (type: bigint) - 1 (_col0 pmod UDFToLong(8)) (type: bigint) + 1 (_col0 pmod 8) (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE Map Join Operator diff --git ql/src/test/results/clientpositive/vectorization_short_regress.q.out ql/src/test/results/clientpositive/vectorization_short_regress.q.out index 7691dda..e83eeef 100644 --- ql/src/test/results/clientpositive/vectorization_short_regress.q.out +++ ql/src/test/results/clientpositive/vectorization_short_regress.q.out @@ -2294,7 +2294,7 @@ STAGE PLANS: alias: alltypesorc Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(ctimestamp1) <> 0.0) and (((-257 <> UDFToInteger(ctinyint)) and cstring1 regexp '.*ss' and (-3.0 < UDFToDouble(ctimestamp1)) and cboolean2 is not null) or (UDFToDouble(ctimestamp2) = -5.0) or ((UDFToDouble(ctimestamp1) < 0.0) and (cstring2 like '%b%')) or (cdouble = UDFToDouble(cint)) or (cboolean1 is null and (cfloat < UDFToFloat(cint))))) (type: boolean) + predicate: ((UDFToDouble(ctimestamp1) <> 0.0) and (((-257 <> UDFToInteger(ctinyint)) and cboolean2 is not null and cstring1 regexp '.*ss' and (-3.0 < UDFToDouble(ctimestamp1))) or (UDFToDouble(ctimestamp2) = -5.0) or ((UDFToDouble(ctimestamp1) < 0.0) and (cstring2 like '%b%')) or (cdouble = UDFToDouble(cint)) or (cboolean1 is null and (cfloat < UDFToFloat(cint))))) (type: boolean) Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cstring1 (type: string), ctimestamp1 (type: timestamp), cint (type: int), csmallint (type: smallint), ctinyint (type: tinyint), cfloat (type: float), cdouble (type: double) @@ -2630,7 +2630,7 @@ STAGE PLANS: alias: alltypesorc Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((cdouble < UDFToDouble(csmallint)) and (cboolean2 = cboolean1) and (UDFToDouble(cbigint) <= -863.257)) or ((cint >= -257) and (cboolean1 >= 1) and cstring1 is not null) or cstring2 regexp 'b' or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null)) and cboolean1 is not null) (type: boolean) + predicate: ((((cdouble < UDFToDouble(csmallint)) and (cboolean2 = cboolean1) and (UDFToDouble(cbigint) <= -863.257)) or ((cint >= -257) and cstring1 is not null and (cboolean1 >= 1)) or cstring2 regexp 'b' or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null)) and cboolean1 is not null) (type: boolean) Statistics: Num rows: 10239 Data size: 2201421 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cboolean1 (type: boolean), cfloat (type: float), cbigint (type: bigint), cint (type: int), cdouble (type: double), ctinyint (type: tinyint), csmallint (type: smallint)