diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java index 3a1897f4aa..ba1b2f9cff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java @@ -1138,8 +1138,15 @@ public HiveSubQRemoveRelBuilder join(JoinRelType joinType, RexNode condition, default: postCondition = condition; } - join = correlateFactory.createCorrelate(left.rel, right.rel, id, - requiredColumns, SemiJoinType.of(joinType)); + if(joinType == JoinRelType.INNER) { + join = correlateFactory.createCorrelate(left.rel, right.rel, id, + requiredColumns, SemiJoinType.SEMI); + } + else { + join = correlateFactory.createCorrelate(left.rel, right.rel, id, + requiredColumns, SemiJoinType.of(joinType)); + + } } else { join = joinFactory.createJoin(left.rel, right.rel, condition, variablesSet, joinType, false); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java index 62125f0fb7..276cbee8b7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java @@ -61,8 +61,10 @@ import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalCorrelate; import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalIntersect; import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.logical.LogicalUnion; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.rules.FilterJoinRule; @@ -77,10 +79,12 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitorImpl; +import org.apache.calcite.sql.SemiJoinType; import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; @@ -92,6 +96,7 @@ import org.apache.calcite.util.Bug; import org.apache.calcite.util.Holder; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Pair; import org.apache.calcite.util.ReflectUtil; @@ -102,9 +107,13 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttleImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIntersect; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -122,6 +131,23 @@ import com.google.common.collect.Sets; import com.google.common.collect.SortedSetMultimap; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.Set; +import java.util.SortedMap; +import java.util.Stack; +import java.util.TreeMap; +import java.util.TreeSet; +import javax.annotation.Nonnull; + /** * NOTE: this whole logic is replicated from Calcite's RelDecorrelator * and is exteneded to make it suitable for HIVE @@ -176,6 +202,8 @@ private final HashSet generatedCorRels = Sets.newHashSet(); + private Stack valueGen = new Stack(); + //~ Constructors ----------------------------------------------------------- private HiveRelDecorrelator ( @@ -262,6 +290,8 @@ private RelNode decorrelate(RelNode root) { return planner2.findBestExp(); } + assert(valueGen.isEmpty()); + return root; } @@ -319,8 +349,14 @@ public RelNode removeCorrelationViaRule(RelNode root) { return planner.findBestExp(); } + protected RexNode decorrelateExpr(RexNode exp, boolean valueGenerator) { + DecorrelateRexShuttle shuttle = new DecorrelateRexShuttle(); + shuttle.setValueGenerator(valueGenerator); + return exp.accept(shuttle); + } protected RexNode decorrelateExpr(RexNode exp) { DecorrelateRexShuttle shuttle = new DecorrelateRexShuttle(); + shuttle.setValueGenerator(false); return exp.accept(shuttle); } @@ -1106,7 +1142,11 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel) { try { findCorrelationEquivalent(correlation, ((Filter) rel).getCondition()); } catch (Util.FoundOne e) { - map.put(def, (Integer) e.getNode()); + // we need to keep predicate kind e.g. EQUAL or NOT EQUAL + // so that later while decorrelating LogicalCorrelate appropriate join predicate + // is generated + def.setPredicateKind((SqlKind)((Pair)e.getNode()).getValue()); + map.put(def, (Integer)((Pair) e.getNode()).getKey()); } } // If all correlation variables are now satisfied, skip creating a value @@ -1145,16 +1185,22 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel) { private void findCorrelationEquivalent(CorRef correlation, RexNode e) throws Util.FoundOne { switch (e.getKind()) { - case EQUALS: + // for now only EQUAL and NOT EQUAL corr predicates are optimized + case NOT_EQUALS: + if((boolean)valueGen.peek()) { + // we will need value generator + break; + } + case EQUALS: final RexCall call = (RexCall) e; final List operands = call.getOperands(); if (references(operands.get(0), correlation) && operands.get(1) instanceof RexInputRef) { - throw new Util.FoundOne(((RexInputRef) operands.get(1)).getIndex()); + throw new Util.FoundOne(Pair.of(((RexInputRef) operands.get(1)).getIndex(), e.getKind())); } if (references(operands.get(1), correlation) && operands.get(0) instanceof RexInputRef) { - throw new Util.FoundOne(((RexInputRef) operands.get(0)).getIndex()); + throw new Util.FoundOne(Pair.of(((RexInputRef) operands.get(0)).getIndex(), e.getKind())); } break; case AND: @@ -1223,17 +1269,38 @@ public Frame decorrelateRel(HiveFilter rel) throws SemanticException { return null; } + Frame oldInputFrame = frame; // If this LogicalFilter has correlated reference, create value generator // and produce the correlated variables in the new output. if (cm.mapRefRelToCorRef.containsKey(rel)) { frame = decorrelateInputWithValueGenerator(rel); } - // Replace the filter expression to reference output of the join - // Map filter to the new filter over join - relBuilder.push(frame.r).filter( - simplifyComparison(decorrelateExpr(rel.getCondition()))); + boolean valueGenerator = true; + if(frame.r == oldInputFrame.r) { + // this means correated value generator wasn't generated + valueGenerator = false; + } + if(oldInput instanceof LogicalCorrelate && ((LogicalCorrelate) oldInput).getJoinType() == SemiJoinType.SEMI + && !cm.mapRefRelToCorRef.containsKey(rel)) { + // this conditions need to be pushed into semi-join since this condition + // corresponds to IN + HiveSemiJoin join = ((HiveSemiJoin)frame.r); + final List conditions = new ArrayList<>(); + RexNode joinCond = join.getCondition(); + conditions.add(joinCond); + conditions.add(rel.getCondition()); + final RexNode condition = + RexUtil.composeConjunction(rexBuilder, conditions, false); + RelNode newRel = HiveSemiJoin.getSemiJoin(frame.r.getCluster(), frame.r.getTraitSet(), join.getLeft(), join.getRight(), + condition,join.getLeftKeys(), join.getRightKeys()); + return register(rel, newRel, frame.oldToNewOutputs, frame.corDefOutputs); + } + // Replace the filter expression to reference output of the join + // Map filter to the new filter over join + relBuilder.push(frame.r).filter( + (decorrelateExpr(rel.getCondition(), valueGenerator))); // Filter does not change the input ordering. // Filter rel does not permute the input. // All corvars produced by filter will have the same output positions in the @@ -1243,39 +1310,6 @@ public Frame decorrelateRel(HiveFilter rel) throws SemanticException { } } - private RexNode simplifyComparison(RexNode op) { - switch(op.getKind()) { - case EQUALS: - case GREATER_THAN: - case GREATER_THAN_OR_EQUAL: - case LESS_THAN: - case LESS_THAN_OR_EQUAL: - case NOT_EQUALS: - RexCall e = (RexCall) op; - final List operands = new ArrayList<>(e.operands); - - // Simplify "x x" - final RexNode o0 = operands.get(0); - final RexNode o1 = operands.get(1); - // this should only be called when we are creating filter (decorrelate filter) - // since in that case null/unknown is treated as false we don't care about - // nullability of operands and will always rewrite op=op to op is not null - if (RexUtil.eq(o0, o1) ) - switch (e.getKind()) { - case EQUALS: - case GREATER_THAN_OR_EQUAL: - case LESS_THAN_OR_EQUAL: - // "x = x" simplifies to "x is not null" (similarly <= and >=) - return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, o0); - default: - // "x != x" simplifies to "false" (similarly < and >) - return rexBuilder.makeLiteral(false); - } - } - return op; - } - - /** * Rewrite LogicalFilter. * @@ -1312,9 +1346,31 @@ public Frame decorrelateRel(LogicalFilter rel) { } + boolean valueGenerator = true; + if(frame.r == oldInput) { + // this means correated value generator wasn't generated + valueGenerator = false; + } + + if(oldInput instanceof LogicalCorrelate && ((LogicalCorrelate) oldInput).getJoinType() == SemiJoinType.SEMI + && !cm.mapRefRelToCorRef.containsKey(rel)) { + // this conditions need to be pushed into semi-join since this condition + // corresponds to IN + HiveSemiJoin join = ((HiveSemiJoin)frame.r); + final List conditions = new ArrayList<>(); + RexNode joinCond = join.getCondition(); + conditions.add(joinCond); + conditions.add(rel.getCondition()); + final RexNode condition = + RexUtil.composeConjunction(rexBuilder, conditions, false); + RelNode newRel = HiveSemiJoin.getSemiJoin(frame.r.getCluster(), frame.r.getTraitSet(), join.getLeft(), join.getRight(), + condition,join.getLeftKeys(), join.getRightKeys()); + return register(rel, newRel, frame.oldToNewOutputs, frame.corDefOutputs); + } + // Replace the filter expression to reference output of the join // Map filter to the new filter over join - relBuilder.push(frame.r).filter(decorrelateExpr(rel.getCondition())); + relBuilder.push(frame.r).filter(decorrelateExpr(rel.getCondition(), valueGenerator)); // Filter does not change the input ordering. @@ -1344,6 +1400,9 @@ public Frame decorrelateRel(LogicalCorrelate rel) { final RelNode oldLeft = rel.getInput(0); final RelNode oldRight = rel.getInput(1); + boolean mightRequireValueGen = new findIfValueGenRequired().traverse(oldRight); + valueGen.push(mightRequireValueGen); + final Frame leftFrame = getInvoke(oldLeft, rel); final Frame rightFrame = getInvoke(oldRight, rel); @@ -1380,11 +1439,24 @@ public Frame decorrelateRel(LogicalCorrelate rel) { } final int newLeftPos = leftFrame.oldToNewOutputs.get(corDef.field); final int newRightPos = rightOutput.getValue(); - conditions.add( - rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, - RexInputRef.of(newLeftPos, newLeftOutput), - new RexInputRef(newLeftFieldCount + newRightPos, - newRightOutput.get(newRightPos).getType()))); + if(corDef.getPredicateKind() == SqlKind.NOT_EQUALS) { + conditions.add( + rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS, + RexInputRef.of(newLeftPos, newLeftOutput), + new RexInputRef(newLeftFieldCount + newRightPos, + newRightOutput.get(newRightPos).getType()))); + + } + else { + assert(corDef.getPredicateKind() == null + || corDef.getPredicateKind() == SqlKind.EQUALS); + conditions.add( + rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, + RexInputRef.of(newLeftPos, newLeftOutput), + new RexInputRef(newLeftFieldCount + newRightPos, + newRightOutput.get(newRightPos).getType()))); + + } // remove this cor var from output position mapping corDefOutputs.remove(corDef); @@ -1408,24 +1480,51 @@ public Frame decorrelateRel(LogicalCorrelate rel) { int oldLeftFieldCount = oldLeft.getRowType().getFieldCount(); int oldRightFieldCount = oldRight.getRowType().getFieldCount(); - assert rel.getRowType().getFieldCount() - == oldLeftFieldCount + oldRightFieldCount; + + // This assertion doesn't hold anymore since rel could be semi-join + //assert rel.getRowType().getFieldCount() + // == oldLeftFieldCount + oldRightFieldCount; // Left input positions are not changed. mapOldToNewOutputs.putAll(leftFrame.oldToNewOutputs); - // Right input positions are shifted by newLeftFieldCount. - for (int i = 0; i < oldRightFieldCount; i++) { - mapOldToNewOutputs.put( - i + oldLeftFieldCount, - rightFrame.oldToNewOutputs.get(i) + newLeftFieldCount); - } final RexNode condition = RexUtil.composeConjunction(rexBuilder, conditions, false); - RelNode newJoin = - LogicalJoin.create(leftFrame.r, rightFrame.r, condition, - ImmutableSet.of(), rel.getJoinType().toJoinType()); + RelNode newJoin = null; + + // this indicates original query was either correlated EXISTS or IN + if(rel.getJoinType() == SemiJoinType.SEMI) { + final List leftKeys = new ArrayList(); + final List rightKeys = new ArrayList(); + //newJoin = HiveSemiJoin.getSemiJoin(rel.getCluster(), rel.getTraitSet(), leftFrame.r, rightFrame.r, + //condition, ImmutableIntList.of(), ImmutableIntList.of()); + + RelNode[] inputRels = new RelNode[] { leftFrame.r, rightFrame.r}; + //List leftJoinKeys = new ArrayList(); + //List rightJoinKeys = new ArrayList(); + //RexNode updatedJoinCond = HiveCalciteUtil.projectNonColumnEquiConditions( + // HiveRelFactories.HIVE_PROJECT_FACTORY, inputRels, leftJoinKeys, rightJoinKeys, 0, + // leftKeys, rightKeys); + newJoin = HiveSemiJoin.getSemiJoin(rel.getCluster(), rel.getCluster().traitSetOf(HiveRelNode.CONVENTION), + leftFrame.r, rightFrame.r, condition, ImmutableIntList.copyOf(leftKeys), + ImmutableIntList.copyOf(rightKeys)); + + } + else { + // Right input positions are shifted by newLeftFieldCount. + for (int i = 0; i < oldRightFieldCount; i++) { + mapOldToNewOutputs.put( + i + oldLeftFieldCount, + rightFrame.oldToNewOutputs.get(i) + newLeftFieldCount); + } + + newJoin = LogicalJoin.create(leftFrame.r, rightFrame.r, condition, + ImmutableSet.of(), rel.getJoinType().toJoinType()); + + } + + valueGen.pop(); return register(rel, newJoin, mapOldToNewOutputs, corDefOutputs); } @@ -1575,7 +1674,7 @@ private RexInputRef getNewForOldInputRef(RexInputRef oldInputRef) { int newLocalOrdinal = oldLocalOrdinal; if (!frame.oldToNewOutputs.isEmpty()) { - newLocalOrdinal = frame.oldToNewOutputs.get(oldLocalOrdinal); + newLocalOrdinal = frame.oldToNewOutputs.get(oldLocalOrdinal); } newOrdinal += newLocalOrdinal; @@ -1819,7 +1918,66 @@ private static RelNode stripHep(RelNode rel) { /** Shuttle that decorrelates. */ private class DecorrelateRexShuttle extends RexShuttle { + private boolean valueGenerator; + public void setValueGenerator(boolean valueGenerator) { + this.valueGenerator = valueGenerator; + } + + // DecorrelateRexShuttle ends up decorrelating expressions cor.col1 <> $4 + // to $4=$4 if value generator is not generated, $4<>$4 is further simplified + // to false. This is wrong and messes up the whole tree. To prevent this visitCall + // is overridden to rewrite/simply such predicates to is not null. + // we also need to take care that we do this only for correlated predicates and + // not user specified explicit predicates + // TODO: This code should be removed once CALCITE-1851 is fixed and + // there is support of not equal + @Override public RexNode visitCall(final RexCall call) { + if(!valueGenerator) { + switch (call.getKind()) { + case EQUALS: + case NOT_EQUALS: + final List operands = new ArrayList<>(call.operands); + RexNode o0 = operands.get(0); + RexNode o1 = operands.get(1); + boolean isCorrelated = false; + if (o0 instanceof RexFieldAccess && (cm.mapFieldAccessToCorRef.get(o0) != null)) { + o0 = decorrFieldAccess((RexFieldAccess) o0); + isCorrelated = true; + + } + if (o1 instanceof RexFieldAccess && (cm.mapFieldAccessToCorRef.get(o1) != null)) { + o1 = decorrFieldAccess((RexFieldAccess) o1); + isCorrelated = true; + } + if (isCorrelated && RexUtil.eq(o0, o1)) { + return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, o0); + } + + final List newOperands = new ArrayList<>(); + newOperands.add(o0); + newOperands.add(o1); + boolean[] update = { false }; + List clonedOperands = visitList(newOperands, update); + + return relBuilder.call(call.getOperator(), clonedOperands); + } + } + return super.visitCall(call); + } + @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { + return decorrFieldAccess(fieldAccess); + } + + @Override public RexNode visitInputRef(RexInputRef inputRef) { + final RexInputRef ref = getNewForOldInputRef(inputRef); + if (ref.getIndex() == inputRef.getIndex() + && ref.getType() == inputRef.getType()) { + return inputRef; // re-use old object, to prevent needless expr cloning + } + return ref; + } + private RexNode decorrFieldAccess(RexFieldAccess fieldAccess) { int newInputOutputOffset = 0; for (RelNode input : currentRel.getInputs()) { final Frame frame = map.get(input); @@ -1834,7 +1992,7 @@ private static RelNode stripHep(RelNode rel) { // This input rel does produce the cor var referenced. // Assume fieldAccess has the correct type info. return new RexInputRef(newInputPos + newInputOutputOffset, - frame.r.getRowType().getFieldList().get(newInputPos) + frame.r.getRowType().getFieldList().get(newInputPos) .getType()); } } @@ -1848,15 +2006,6 @@ private static RelNode stripHep(RelNode rel) { } return fieldAccess; } - - @Override public RexNode visitInputRef(RexInputRef inputRef) { - final RexInputRef ref = getNewForOldInputRef(inputRef); - if (ref.getIndex() == inputRef.getIndex() - && ref.getType() == inputRef.getType()) { - return inputRef; // re-use old object, to prevent needless expr cloning - } - return ref; - } } /** Shuttle that removes correlations. */ @@ -2881,10 +3030,12 @@ public CorDef def() { static class CorDef implements Comparable { public final CorrelationId corr; public final int field; + private SqlKind predicateKind; CorDef(CorrelationId corr, int field) { this.corr = corr; this.field = field; + this.predicateKind = null; } @Override public String toString() { @@ -2909,6 +3060,13 @@ public int compareTo(@Nonnull CorDef o) { } return Integer.compare(field, o.field); } + public SqlKind getPredicateKind() { + return predicateKind; + } + public void setPredicateKind(SqlKind predKind) { + this.predicateKind = predKind; + + } } /** A map of the locations of @@ -2986,6 +3144,107 @@ public boolean hasCorrelation() { } } + private static class findIfValueGenRequired extends HiveRelShuttleImpl { + private boolean mightRequireValueGen ; + findIfValueGenRequired() { this.mightRequireValueGen = true; } + + private boolean hasRexOver(List projects) { + for(RexNode expr : projects) { + if(expr instanceof RexOver) { + return true; + } + } + return false; + } + @Override public RelNode visit(HiveJoin rel) { + mightRequireValueGen = true; + return rel; + } + public RelNode visit(HiveSortLimit rel) { + mightRequireValueGen = true; + return rel; + } + public RelNode visit(HiveUnion rel) { + mightRequireValueGen = true; + return rel; + } + public RelNode visit(LogicalUnion rel) { + mightRequireValueGen = true; + return rel; + } + public RelNode visit(LogicalIntersect rel) { + mightRequireValueGen = true; + return rel; + } + + public RelNode visit(HiveIntersect rel) { + mightRequireValueGen = true; + return rel; + } + + @Override public RelNode visit(LogicalJoin rel) { + mightRequireValueGen = true; + return rel; + } + @Override public RelNode visit(HiveProject rel) { + if(!(hasRexOver(((HiveProject)rel).getProjects()))) { + mightRequireValueGen = false; + return super.visit(rel); + } + else { + mightRequireValueGen = true; + return rel; + } + } + @Override public RelNode visit(LogicalProject rel) { + if(!(hasRexOver(((LogicalProject)rel).getProjects()))) { + mightRequireValueGen = false; + return super.visit(rel); + } + else { + mightRequireValueGen = true; + return rel; + } + } + @Override public RelNode visit(HiveAggregate rel) { + // if there are aggregate functions or grouping sets we will need + // value generator + if((((HiveAggregate)rel).getAggCallList().isEmpty() == true + && ((HiveAggregate)rel).indicator == false)) { + this.mightRequireValueGen = false; + return super.visit(rel); + } + else { + // need to reset to true in case previous aggregate/project + // has set it to false + this.mightRequireValueGen = true; + return rel; + } + } + @Override public RelNode visit(LogicalAggregate rel) { + if((((LogicalAggregate)rel).getAggCallList().isEmpty() == true + && ((LogicalAggregate)rel).indicator == false)) { + this.mightRequireValueGen = false; + return super.visit(rel); + } + else { + // need to reset to true in case previous aggregate/project + // has set it to false + this.mightRequireValueGen = true; + return rel; + } + } + @Override public RelNode visit(LogicalCorrelate rel) { + // this means we are hitting nested subquery so don't + // need to go further + return rel; + } + + public boolean traverse(RelNode root) { + root.accept(this); + return mightRequireValueGen; + } + } /** Builds a {@link org.apache.calcite.sql2rel.RelDecorrelator.CorelMap}. */ private static class CorelMapBuilder extends HiveRelShuttleImpl { final SortedMap mapCorToCorRel = diff --git a/ql/src/test/queries/clientpositive/subquery_exists.q b/ql/src/test/queries/clientpositive/subquery_exists.q index 19c42f0c29..0dfab7a1a3 100644 --- a/ql/src/test/queries/clientpositive/subquery_exists.q +++ b/ql/src/test/queries/clientpositive/subquery_exists.q @@ -85,27 +85,3 @@ explain select * from t where exists (select count(*) from src where 1=2); select * from t where exists (select count(*) from src where 1=2); drop table t; - -drop table if exists tx1; -create table tx1 (a integer,b integer); -insert into tx1 values (1, 1), - (1, 2), - (1, 3); - -select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b); -explain select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b); - -drop table tx1; - -create table t1(i int, j int); -insert into t1 values(4,1); - -create table t2(i int, j int); -insert into t2 values(4,2),(4,3),(4,5); - -explain select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j); -select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j); -drop table t1; -drop table t2; diff --git a/ql/src/test/queries/clientpositive/subquery_in.q b/ql/src/test/queries/clientpositive/subquery_in.q index 4ba170a706..33cc2feb97 100644 --- a/ql/src/test/queries/clientpositive/subquery_in.q +++ b/ql/src/test/queries/clientpositive/subquery_in.q @@ -74,25 +74,47 @@ from part b where b.p_size in ; -- distinct, corr -explain -select * -from src b +explain +select * +from src b where b.key in - (select distinct a.key - from src a + (select distinct a.key + from src a where b.value = a.value and a.key > '9' ) ; -select * -from src b +select * +from src b where b.key in - (select distinct a.key - from src a + (select distinct a.key + from src a where b.value = a.value and a.key > '9' ) ; +-- corr, non equi predicate, should not have a join with outer to generate +-- corr values +explain +select * +from src b +where b.key in + (select distinct a.key + from src a + where b.value <> a.key and a.key > '9' + ) +; + +select * +from src b +where b.key in + (select distinct a.key + from src a + where b.value <> a.key and a.key > '9' + ) +; + + -- non agg, non corr, windowing select p_mfgr, p_name, p_size from part @@ -267,3 +289,13 @@ select * from t where i IN (select sum(i) from tt where tt.j = t.j); drop table t; drop table tt; + +-- since inner query has aggregate it will be joined with outer to get all possible corrrelated values +explain select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name); +select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name); + +-- inner query has join so should have a join with outer query to fetch all corr values +explain select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name); +select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name); + + diff --git a/ql/src/test/results/clientpositive/constprog_partitioner.q.out b/ql/src/test/results/clientpositive/constprog_partitioner.q.out index 87618df902..98b896c87a 100644 --- a/ql/src/test/results/clientpositive/constprog_partitioner.q.out +++ b/ql/src/test/results/clientpositive/constprog_partitioner.q.out @@ -107,22 +107,22 @@ STAGE PLANS: alias: lineitem Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((l_linenumber = l_linenumber) and (l_shipmode = 'AIR')) (type: boolean) - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean) + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: l_orderkey (type: int), l_linenumber (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 6e55acf0d8..9b22beb876 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -1683,11 +1683,11 @@ Stage-0 Stage-1 Reducer 2 llap File Output Operator [FS_19] - Select Operator [SEL_18] (rows=434 width=178) + Select Operator [SEL_18] (rows=366 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_17] (rows=434 width=178) + Filter Operator [FIL_17] (rows=366 width=179) predicate:_col3 is null - Merge Join Operator [MERGEJOIN_22] (rows=500 width=178) + Merge Join Operator [MERGEJOIN_22] (rows=500 width=179) Conds:RS_14._col1=RS_15._col0(Left Outer),Output:["_col0","_col1","_col3"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_14] @@ -1699,21 +1699,21 @@ Stage-0 <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_15] PartitionCols:_col0 - Select Operator [SEL_13] (rows=41 width=95) + Select Operator [SEL_13] (rows=83 width=95) Output:["_col0","_col1"] - Group By Operator [GBY_12] (rows=41 width=91) + Group By Operator [GBY_12] (rows=83 width=91) Output:["_col0"],keys:_col1 - Select Operator [SEL_8] (rows=41 width=178) + Select Operator [SEL_8] (rows=83 width=178) Output:["_col1"] - Group By Operator [GBY_7] (rows=41 width=178) + Group By Operator [GBY_7] (rows=83 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_6] PartitionCols:_col0 - Group By Operator [GBY_5] (rows=41 width=178) + Group By Operator [GBY_5] (rows=83 width=178) Output:["_col0","_col1"],keys:value, key - Filter Operator [FIL_21] (rows=83 width=178) - predicate:((value = value) and (value > 'val_2')) + Filter Operator [FIL_21] (rows=166 width=178) + predicate:(value > 'val_2') Please refer to the previous TableScan [TS_0] PREHOOK: query: explain select * @@ -1747,11 +1747,11 @@ Stage-0 Stage-1 Reducer 3 llap File Output Operator [FS_18] - Select Operator [SEL_17] (rows=234 width=178) + Select Operator [SEL_17] (rows=183 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=234 width=178) + Filter Operator [FIL_16] (rows=183 width=179) predicate:_col4 is null - Merge Join Operator [MERGEJOIN_21] (rows=250 width=178) + Merge Join Operator [MERGEJOIN_21] (rows=250 width=179) Conds:RS_13._col0, _col1=RS_14._col0, _col1(Left Outer),Output:["_col0","_col1","_col4"] <-Reducer 2 [ONE_TO_ONE_EDGE] llap FORWARD [RS_13] @@ -1770,17 +1770,17 @@ Stage-0 <-Reducer 4 [ONE_TO_ONE_EDGE] llap FORWARD [RS_14] PartitionCols:_col0, _col1 - Select Operator [SEL_12] (rows=20 width=182) + Select Operator [SEL_12] (rows=83 width=182) Output:["_col0","_col1","_col2"] - Group By Operator [GBY_11] (rows=20 width=178) + Group By Operator [GBY_11] (rows=83 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_10] PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=20 width=178) + Group By Operator [GBY_9] (rows=83 width=178) Output:["_col0","_col1"],keys:key, value - Filter Operator [FIL_20] (rows=41 width=178) - predicate:((key = key) and (value = value) and (value > 'val_12')) + Filter Operator [FIL_20] (rows=166 width=178) + predicate:((value > 'val_12') and key is not null) Please refer to the previous TableScan [TS_0] PREHOOK: query: create view cv1 as @@ -1822,7 +1822,7 @@ Stage-0 Stage-1 Reducer 2 llap File Output Operator [FS_11] - Merge Join Operator [MERGEJOIN_16] (rows=32 width=178) + Merge Join Operator [MERGEJOIN_16] (rows=134 width=178) Conds:RS_7._col0, _col1=RS_8._col0, _col1(Left Semi),Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_7] @@ -1834,12 +1834,12 @@ Stage-0 <-Map 3 [SIMPLE_EDGE] llap SHUFFLE [RS_8] PartitionCols:_col0, _col1 - Group By Operator [GBY_6] (rows=20 width=178) + Group By Operator [GBY_6] (rows=83 width=178) Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_4] (rows=41 width=178) + Select Operator [SEL_4] (rows=166 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_15] (rows=41 width=178) - predicate:((key = key) and (value = value) and (value > 'val_9')) + Filter Operator [FIL_15] (rows=166 width=178) + predicate:((value > 'val_9') and key is not null) TableScan [TS_2] (rows=500 width=178) default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -1872,7 +1872,7 @@ Stage-0 Stage-1 Reducer 2 llap File Output Operator [FS_11] - Merge Join Operator [MERGEJOIN_16] (rows=32 width=178) + Merge Join Operator [MERGEJOIN_16] (rows=134 width=178) Conds:RS_7._col0, _col1=RS_8._col0, _col1(Left Semi),Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_7] @@ -1884,12 +1884,12 @@ Stage-0 <-Map 3 [SIMPLE_EDGE] llap SHUFFLE [RS_8] PartitionCols:_col0, _col1 - Group By Operator [GBY_6] (rows=20 width=178) + Group By Operator [GBY_6] (rows=83 width=178) Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_4] (rows=41 width=178) + Select Operator [SEL_4] (rows=166 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_15] (rows=41 width=178) - predicate:((key = key) and (value = value) and (value > 'val_9')) + Filter Operator [FIL_15] (rows=166 width=178) + predicate:((value > 'val_9') and key is not null) TableScan [TS_2] (rows=500 width=178) default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -1965,12 +1965,12 @@ Stage-0 <-Map 6 [SIMPLE_EDGE] llap SHUFFLE [RS_19] PartitionCols:_col0, _col1 - Group By Operator [GBY_17] (rows=2 width=8) + Group By Operator [GBY_17] (rows=4 width=8) Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_12] (rows=7 width=8) + Select Operator [SEL_12] (rows=14 width=8) Output:["_col0","_col1"] - Filter Operator [FIL_30] (rows=7 width=96) - predicate:((l_linenumber = l_linenumber) and (l_shipmode = 'AIR')) + Filter Operator [FIL_30] (rows=14 width=96) + predicate:((l_shipmode = 'AIR') and l_linenumber is not null) TableScan [TS_10] (rows=100 width=96) default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_linenumber","l_shipmode"] <-Reducer 3 [SIMPLE_EDGE] llap @@ -2258,14 +2258,14 @@ Stage-0 File Output Operator [FS_26] Select Operator [SEL_25] (rows=13 width=223) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_24] (rows=13 width=227) + Filter Operator [FIL_24] (rows=13 width=231) predicate:(not CASE WHEN ((_col4 = 0)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) - Merge Join Operator [MERGEJOIN_32] (rows=26 width=227) + Merge Join Operator [MERGEJOIN_32] (rows=26 width=230) Conds:RS_21._col0, _col1=RS_22._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col8"] <-Reducer 2 [SIMPLE_EDGE] llap SHUFFLE [RS_21] PartitionCols:_col0, _col1 - Merge Join Operator [MERGEJOIN_31] (rows=26 width=226) + Merge Join Operator [MERGEJOIN_31] (rows=26 width=229) Conds:RS_18._col1=RS_19._col0(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_18] @@ -2277,36 +2277,36 @@ Stage-0 <-Reducer 4 [ONE_TO_ONE_EDGE] llap FORWARD [RS_19] PartitionCols:_col0 - Group By Operator [GBY_7] (rows=1 width=114) + Group By Operator [GBY_7] (rows=2 width=114) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_6] PartitionCols:_col0 - Group By Operator [GBY_5] (rows=1 width=114) + Group By Operator [GBY_5] (rows=2 width=114) Output:["_col0","_col1","_col2"],aggregations:["count()","count(p_name)"],keys:p_mfgr - Select Operator [SEL_4] (rows=4 width=223) + Select Operator [SEL_4] (rows=8 width=223) Output:["p_name","p_mfgr"] - Filter Operator [FIL_29] (rows=4 width=223) - predicate:((p_mfgr = p_mfgr) and (p_size < 10)) + Filter Operator [FIL_29] (rows=8 width=223) + predicate:((p_size < 10) and p_mfgr is not null) Please refer to the previous TableScan [TS_0] <-Reducer 5 [ONE_TO_ONE_EDGE] llap FORWARD [RS_22] PartitionCols:_col0, _col1 - Select Operator [SEL_17] (rows=2 width=223) + Select Operator [SEL_17] (rows=4 width=223) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_16] (rows=2 width=219) + Filter Operator [FIL_16] (rows=4 width=219) predicate:_col0 is not null - Group By Operator [GBY_14] (rows=2 width=219) + Group By Operator [GBY_14] (rows=4 width=219) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_13] PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=2 width=219) + Group By Operator [GBY_12] (rows=4 width=219) Output:["_col0","_col1"],keys:p_name, p_mfgr - Select Operator [SEL_11] (rows=4 width=223) + Select Operator [SEL_11] (rows=8 width=223) Output:["p_name","p_mfgr"] - Filter Operator [FIL_30] (rows=4 width=223) - predicate:((p_mfgr = p_mfgr) and (p_size < 10)) + Filter Operator [FIL_30] (rows=8 width=223) + predicate:((p_size < 10) and p_mfgr is not null) Please refer to the previous TableScan [TS_0] PREHOOK: query: explain select p_name, p_size @@ -2465,7 +2465,7 @@ Stage-0 Select Operator [SEL_12] (rows=1 width=114) Output:["_col0","_col1"] Filter Operator [FIL_40] (rows=1 width=114) - predicate:(((_col2 - _col1) > 600.0) and (_col1 = _col1)) + predicate:(((_col2 - _col1) > 600.0) and _col1 is not null) Group By Operator [GBY_10] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 <-Map 1 [SIMPLE_EDGE] llap @@ -2484,7 +2484,7 @@ Stage-0 Select Operator [SEL_24] (rows=1 width=110) Output:["_col0","_col1"] Filter Operator [FIL_41] (rows=1 width=114) - predicate:(((_col2 - _col1) > 600.0) and (_col1 = _col1)) + predicate:(((_col2 - _col1) > 600.0) and _col1 is not null) Group By Operator [GBY_22] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 <-Map 1 [SIMPLE_EDGE] llap diff --git a/ql/src/test/results/clientpositive/llap/subquery_exists.q.out b/ql/src/test/results/clientpositive/llap/subquery_exists.q.out index e206f0851e..759b6a02da 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_exists.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_exists.q.out @@ -25,7 +25,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -33,39 +34,28 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((key = key) and (value = value) and (value > 'val_9')) (type: boolean) - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((value > 'val_9') and key is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string), value (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -78,14 +68,35 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 32 Data size: 5696 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 66 Data size: 11748 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 32 Data size: 5696 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 66 Data size: 11748 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col0 is not null and _col1 is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -229,7 +240,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -237,16 +249,19 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs Map 3 @@ -257,20 +272,16 @@ STAGE PLANS: Filter Operator predicate: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) + Group By Operator + keys: value (type: string) + mode: hash outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -283,14 +294,35 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 202 Data size: 35956 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 202 Data size: 35956 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 125 Data size: 11375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 125 Data size: 11375 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -1061,374 +1093,3 @@ POSTHOOK: query: drop table t POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@t POSTHOOK: Output: default@t -PREHOOK: query: drop table if exists tx1 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table if exists tx1 -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table tx1 (a integer,b integer) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@tx1 -POSTHOOK: query: create table tx1 (a integer,b integer) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@tx1 -PREHOOK: query: insert into tx1 values (1, 1), - (1, 2), - (1, 3) -PREHOOK: type: QUERY -PREHOOK: Output: default@tx1 -POSTHOOK: query: insert into tx1 values (1, 1), - (1, 2), - (1, 3) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@tx1 -POSTHOOK: Lineage: tx1.a EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: tx1.b EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -PREHOOK: query: select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -PREHOOK: type: QUERY -PREHOOK: Input: default@tx1 -#### A masked pattern was here #### -POSTHOOK: query: select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@tx1 -#### A masked pattern was here #### -3 3 -PREHOOK: query: explain select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -PREHOOK: type: QUERY -POSTHOOK: query: explain select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 5 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: u - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) - Group By Operator - keys: a (type: int), b (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 _col0 (type: int), _col1 (type: int) - 1 _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), 3 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col1, _col2, _col3 - residual filter predicates: {(_col3 <> _col1)} - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: int), _col3 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 26 Basic stats: COMPLETE Column stats: NONE - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: drop table tx1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@tx1 -PREHOOK: Output: default@tx1 -POSTHOOK: query: drop table tx1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@tx1 -POSTHOOK: Output: default@tx1 -PREHOOK: query: create table t1(i int, j int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t1 -POSTHOOK: query: create table t1(i int, j int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1 -PREHOOK: query: insert into t1 values(4,1) -PREHOOK: type: QUERY -PREHOOK: Output: default@t1 -POSTHOOK: query: insert into t1 values(4,1) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@t1 -POSTHOOK: Lineage: t1.i EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: t1.j EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -PREHOOK: query: create table t2(i int, j int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t2 -POSTHOOK: query: create table t2(i int, j int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t2 -PREHOOK: query: insert into t2 values(4,2),(4,3),(4,5) -PREHOOK: type: QUERY -PREHOOK: Output: default@t2 -POSTHOOK: query: insert into t2 values(4,2),(4,3),(4,5) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@t2 -POSTHOOK: Lineage: t2.i EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: t2.j EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -Warning: Shuffle Join MERGEJOIN[27][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product -PREHOOK: query: explain select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (XPROD_EDGE), Reducer 6 (XPROD_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 _col0 (type: int), _col1 (type: int) - 1 _col0 (type: int), _col1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 42 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 42 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - residual filter predicates: {(_col1 <> _col2)} - Statistics: Num rows: 3 Data size: 39 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col2 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 39 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 39 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 39 Basic stats: COMPLETE Column stats: NONE - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join MERGEJOIN[27][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product -PREHOOK: query: select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 -#### A masked pattern was here #### -POSTHOOK: query: select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 -#### A masked pattern was here #### -4 1 -PREHOOK: query: drop table t1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t1 -PREHOOK: Output: default@t1 -POSTHOOK: query: drop table t1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t1 -POSTHOOK: Output: default@t1 -PREHOOK: query: drop table t2 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t2 -PREHOOK: Output: default@t2 -POSTHOOK: query: drop table t2 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t2 -POSTHOOK: Output: default@t2 diff --git a/ql/src/test/results/clientpositive/llap/subquery_in.q.out b/ql/src/test/results/clientpositive/llap/subquery_in.q.out index af42131bc2..f908250789 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -164,22 +164,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((key > '9') and (value = value)) (type: boolean) - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -192,10 +192,10 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 66 Data size: 11748 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 134 Data size: 23852 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 66 Data size: 11748 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 134 Data size: 23852 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -463,13 +463,13 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (p_mfgr = p_mfgr) (type: boolean) - Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE + predicate: p_mfgr is not null (type: boolean) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: p_mfgr (type: string), p_size (type: int) sort order: ++ Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs @@ -483,14 +483,14 @@ STAGE PLANS: 0 _col1 (type: string), _col2 (type: int) 1 _col1 (type: string), _col0 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 223 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 223 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 223 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -501,7 +501,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) outputColumnNames: _col2, _col5 - Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -522,25 +522,25 @@ STAGE PLANS: window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (rank_window_0 <= 2) (type: boolean) - Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: string), _col5 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) Reducer 5 Execution mode: llap @@ -550,16 +550,16 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string), _col0 (type: int) sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -591,21 +591,21 @@ Manufacturer#2 almond aquamarine midnight light salmon 2 Manufacturer#3 almond antique misty red olive 1 Manufacturer#4 almond aquamarine yellow dodger mint 7 Manufacturer#5 almond antique sky peru orange 2 -PREHOOK: query: explain -select * -from src b +PREHOOK: query: explain +select * +from src b where b.key in - (select distinct a.key - from src a + (select distinct a.key + from src a where b.value = a.value and a.key > '9' ) PREHOOK: type: QUERY -POSTHOOK: query: explain -select * -from src b +POSTHOOK: query: explain +select * +from src b where b.key in - (select distinct a.key - from src a + (select distinct a.key + from src a where b.value = a.value and a.key > '9' ) POSTHOOK: type: QUERY @@ -643,22 +643,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((key > '9') and (value = value)) (type: boolean) - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -671,10 +671,10 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 66 Data size: 11748 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 134 Data size: 23852 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 66 Data size: 11748 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 134 Data size: 23852 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -686,21 +686,21 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select * -from src b +PREHOOK: query: select * +from src b where b.key in - (select distinct a.key - from src a + (select distinct a.key + from src a where b.value = a.value and a.key > '9' ) PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select * -from src b +POSTHOOK: query: select * +from src b where b.key in - (select distinct a.key - from src a + (select distinct a.key + from src a where b.value = a.value and a.key > '9' ) POSTHOOK: type: QUERY @@ -717,6 +717,148 @@ POSTHOOK: Input: default@src 97 val_97 98 val_98 98 val_98 +PREHOOK: query: explain +select * +from src b +where b.key in + (select distinct a.key + from src a + where b.value <> a.key and a.key > '9' + ) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * +from src b +where b.key in + (select distinct a.key + from src a + where b.value <> a.key and a.key > '9' + ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col3 + residual filter predicates: {(_col1 <> _col3)} + Statistics: Num rows: 134 Data size: 35510 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 134 Data size: 23852 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 134 Data size: 23852 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * +from src b +where b.key in + (select distinct a.key + from src a + where b.value <> a.key and a.key > '9' + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * +from src b +where b.key in + (select distinct a.key + from src a + where b.value <> a.key and a.key > '9' + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +90 val_90 +90 val_90 +90 val_90 +92 val_92 +95 val_95 +95 val_95 +96 val_96 +97 val_97 +97 val_97 +98 val_98 +98 val_98 PREHOOK: query: select p_mfgr, p_name, p_size from part where part.p_size in @@ -1967,18 +2109,18 @@ STAGE PLANS: alias: p Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((p_partkey = p_partkey) and (p_size = p_size)) (type: boolean) - Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (p_partkey is not null and p_size is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_partkey (type: int), p_name (type: string), p_size (type: int) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) sort order: +++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int) - Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -1991,10 +2133,10 @@ STAGE PLANS: 0 _col0 (type: int), _col1 (type: string), _col5 (type: int) 1 _col1 (type: int), _col0 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 3 Data size: 1857 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 8047 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 1857 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 8047 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2006,16 +2148,16 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: int), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int), _col0 (type: string), _col2 (type: int) sort order: +++ Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col2 (type: int) - Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -3275,22 +3417,22 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((p_mfgr = p_mfgr) and (p_size < 10)) (type: boolean) - Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((p_size < 10) and p_mfgr is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 1784 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_mfgr (type: string), p_name (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -3303,15 +3445,15 @@ STAGE PLANS: 0 _col1 (type: string), _col0 (type: string) 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) outputColumnNames: _col1, _col3, _col4 - Statistics: Num rows: 2 Data size: 650 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col3 (type: string), _col4 (type: int) sort order: ++ - Statistics: Num rows: 2 Data size: 650 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Reducer 3 Execution mode: llap @@ -3319,10 +3461,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -5273,3 +5415,386 @@ POSTHOOK: query: drop table tt POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@tt POSTHOOK: Output: default@tt +Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +PREHOOK: query: explain select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Reducer 7 <- Map 6 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col5 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col5 (type: int) + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string), _col5 (type: int) + 1 _col1 (type: string), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + residual filter predicates: {(_col0 <> _col2)} + Statistics: Num rows: 338 Data size: 77402 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int), _col2 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 338 Data size: 77402 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(_col1) + keys: _col2 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) + Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +PREHOOK: query: select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +PREHOOK: query: explain select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) + Reducer 5 <- Map 9 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: string), _col5 (type: int) + sort order: ++ + Map-reduce partition columns: _col4 (type: string), _col5 (type: int) + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 9 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col4 (type: string), _col5 (type: int) + 1 _col1 (type: string), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 198 Data size: 122562 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 198 Data size: 122562 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + residual filter predicates: {(_col2 <> _col0)} + Statistics: Num rows: 338 Data size: 111202 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 338 Data size: 111202 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: string) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col2, _col4 + Statistics: Num rows: 366 Data size: 39528 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col2 (type: string), _col4 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 183 Data size: 19764 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 183 Data size: 19764 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 183 Data size: 19764 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 183 Data size: 19764 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) + Statistics: Num rows: 183 Data size: 19764 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +PREHOOK: query: select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ +110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously +112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +132666 almond aquamarine rose maroon antique Manufacturer#2 Brand#24 SMALL POLISHED NICKEL 25 MED BOX 1698.66 even +144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about +146985 almond aquamarine midnight light salmon Manufacturer#2 Brand#23 MEDIUM BURNISHED COPPER 2 SM CASE 2031.98 s cajole caref +15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu +155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra +17273 almond antique forest lavender goldenrod Manufacturer#3 Brand#35 PROMO ANODIZED TIN 14 JUMBO CASE 1190.27 along the +17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve +191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle +192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir +195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de +33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful +40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s +42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl +45261 almond aquamarine floral ivory bisque Manufacturer#4 Brand#42 SMALL PLATED STEEL 27 WRAP CASE 1206.26 careful +48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i +49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick +65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr +78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith +85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull +86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully +90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out index 96fe17a05a..44b4e2a6ea 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -1665,7 +1665,7 @@ STAGE PLANS: Edges: Reducer 10 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 11 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) @@ -1678,16 +1678,19 @@ STAGE PLANS: TableScan alias: part_null Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Filter Operator + predicate: (p_name is not null and p_type is not null) (type: boolean) Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col4 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col4 (type: string), _col1 (type: string) + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Reduce Output Operator + key expressions: _col4 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col4 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs Map 12 @@ -1696,7 +1699,7 @@ STAGE PLANS: alias: pp Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: p_type is not null (type: boolean) + predicate: (p_brand is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_type (type: string), p_brand (type: string) @@ -1725,7 +1728,7 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: p_type is not null (type: boolean) + predicate: (p_name is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_type (type: string), p_name (type: string) @@ -1745,7 +1748,7 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: p_type is not null (type: boolean) + predicate: (p_brand is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_brand (type: string), p_type (type: string), p_container (type: string) @@ -1812,17 +1815,17 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Inner Join 0 to 1 + Left Semi Join 0 to 1 keys: 0 _col4 (type: string), _col1 (type: string) - 1 _col1 (type: string), _col0 (type: string) + 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 14 Data size: 3217 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1485 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col4 (type: string) sort order: + Map-reduce partition columns: _col4 (type: string) - Statistics: Num rows: 14 Data size: 3217 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1485 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 3 Execution mode: llap @@ -1833,14 +1836,14 @@ STAGE PLANS: keys: 0 _col4 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13 - Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11 + Statistics: Num rows: 7 Data size: 924 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: string), _col4 (type: string) sort order: ++ Map-reduce partition columns: _col3 (type: string), _col4 (type: string) - Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) + Statistics: Num rows: 7 Data size: 924 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) Reducer 4 Execution mode: llap Reduce Operator Tree: @@ -1850,18 +1853,18 @@ STAGE PLANS: keys: 0 _col3 (type: string), _col4 (type: string) 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col16 - Statistics: Num rows: 16 Data size: 3891 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col14 + Statistics: Num rows: 7 Data size: 1016 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col12 = 0)) THEN (false) WHEN (_col12 is null) THEN (false) WHEN (_col16 is not null) THEN (true) WHEN (_col3 is null) THEN (null) WHEN ((_col13 < _col12)) THEN (true) ELSE (false) END) (type: boolean) - Statistics: Num rows: 8 Data size: 1945 Basic stats: COMPLETE Column stats: NONE + predicate: (not CASE WHEN ((_col10 = 0)) THEN (false) WHEN (_col10 is null) THEN (false) WHEN (_col14 is not null) THEN (true) WHEN (_col3 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (true) ELSE (false) END) (type: boolean) + Statistics: Num rows: 4 Data size: 580 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 1945 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 580 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 1945 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 580 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1874,15 +1877,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 2925 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col0 (type: string) + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2925 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1350 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 13 Data size: 2925 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 6 Data size: 1350 Basic stats: COMPLETE Column stats: COMPLETE Reducer 8 Execution mode: llap Reduce Operator Tree: @@ -1981,7 +1985,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1992,62 +1996,62 @@ STAGE PLANS: alias: part_null Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: p_name is not null (type: boolean) + predicate: (p_brand is not null and p_name is not null and p_type is not null) (type: boolean) Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + key expressions: _col3 (type: string), _col4 (type: string) + sort order: ++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: string) Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs Map 4 Map Operator Tree: TableScan alias: part - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (p_brand is not null and p_type is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: p_name (type: string) + keys: p_brand (type: string), p_type (type: string) mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Map 6 Map Operator Tree: TableScan alias: part - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + predicate: p_name is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: p_brand (type: string), p_type (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + expressions: p_name (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + keys: _col0 (type: string) mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -2055,18 +2059,18 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Inner Join 0 to 1 + Left Semi Join 0 to 1 keys: - 0 _col1 (type: string) - 1 _col0 (type: string) + 0 _col3 (type: string), _col4 (type: string) + 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1293 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string), _col4 (type: string) - sort order: ++ - Map-reduce partition columns: _col3 (type: string), _col4 (type: string) - Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 6 Data size: 1293 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 3 Execution mode: llap Reduce Operator Tree: @@ -2074,13 +2078,13 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 keys: - 0 _col3 (type: string), _col4 (type: string) - 1 _col0 (type: string), _col1 (type: string) + 0 _col1 (type: string) + 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2089,15 +2093,20 @@ STAGE PLANS: Execution mode: llap Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 6 Data size: 1176 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -2115,32 +2124,32 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Input: default@part_null #### A masked pattern was here #### -85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull -65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr -86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully +192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h 121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl +85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull +17273 almond antique forest lavender goldenrod Manufacturer#3 Brand#35 PROMO ANODIZED TIN 14 JUMBO CASE 1190.27 along the +49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick +42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl +112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car +40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s +144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about 110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously -191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle +155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra 105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ +48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i +191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle +86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully +15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu +45261 almond aquamarine floral ivory bisque Manufacturer#4 Brand#42 SMALL PLATED STEEL 27 WRAP CASE 1206.26 careful 146985 almond aquamarine midnight light salmon Manufacturer#2 Brand#23 MEDIUM BURNISHED COPPER 2 SM CASE 2031.98 s cajole caref +65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr 132666 almond aquamarine rose maroon antique Manufacturer#2 Brand#24 SMALL POLISHED NICKEL 25 MED BOX 1698.66 even 195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de -90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s -112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car -144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about -17273 almond antique forest lavender goldenrod Manufacturer#3 Brand#35 PROMO ANODIZED TIN 14 JUMBO CASE 1190.27 along the 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve -49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful -48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i -45261 almond aquamarine floral ivory bisque Manufacturer#4 Brand#42 SMALL PLATED STEEL 27 WRAP CASE 1206.26 careful -42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir -15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu -155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra PREHOOK: query: explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type AND part.p_container=part_null.p_container) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN ( select p_name from part where part.p_type = part_null.p_type AND part.p_container=part_null.p_container) AND p_brand NOT IN (select p_container from part where part.p_type = part_null.p_type AND p_brand IN (select p_brand from part pp where part.p_type = pp.p_type)) @@ -2156,7 +2165,7 @@ STAGE PLANS: Edges: Reducer 10 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 11 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) @@ -2169,16 +2178,19 @@ STAGE PLANS: TableScan alias: part_null Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Filter Operator + predicate: (p_container is not null and p_name is not null and p_type is not null) (type: boolean) Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col4 (type: string), _col1 (type: string), _col6 (type: string) - sort order: +++ - Map-reduce partition columns: _col4 (type: string), _col1 (type: string), _col6 (type: string) + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col7 (type: double), _col8 (type: string) + Reduce Output Operator + key expressions: _col4 (type: string), _col1 (type: string), _col6 (type: string) + sort order: +++ + Map-reduce partition columns: _col4 (type: string), _col1 (type: string), _col6 (type: string) + Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs Map 12 @@ -2187,7 +2199,7 @@ STAGE PLANS: alias: pp Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: p_type is not null (type: boolean) + predicate: (p_brand is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_type (type: string), p_brand (type: string) @@ -2216,18 +2228,18 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((p_container = p_container) and (p_type = p_type)) (type: boolean) - Statistics: Num rows: 6 Data size: 1902 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (p_container is not null and p_name is not null and p_type is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_type (type: string), p_name (type: string), p_container (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 951 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 4121 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 3 Data size: 951 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 4121 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Map 7 @@ -2236,7 +2248,7 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: p_type is not null (type: boolean) + predicate: (p_brand is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_brand (type: string), p_type (type: string), p_container (type: string) @@ -2268,7 +2280,7 @@ STAGE PLANS: outputColumnNames: _col1, _col2 Statistics: Num rows: 14 Data size: 2744 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: _col1 (type: string), _col2 (type: string) + keys: _col2 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE @@ -2285,39 +2297,35 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col0 (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: _col0 is not null (type: boolean) Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Reducer 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator condition map: - Inner Join 0 to 1 + Left Semi Join 0 to 1 keys: 0 _col4 (type: string), _col1 (type: string), _col6 (type: string) - 1 _col1 (type: string), _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 3 Data size: 1046 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 2092 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col4 (type: string) sort order: + Map-reduce partition columns: _col4 (type: string) - Statistics: Num rows: 3 Data size: 1046 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 2092 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 3 Execution mode: llap @@ -2328,14 +2336,14 @@ STAGE PLANS: keys: 0 _col4 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col13, _col14 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11 Statistics: Num rows: 7 Data size: 924 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col4 (type: string), _col3 (type: string) + key expressions: _col3 (type: string), _col4 (type: string) sort order: ++ - Map-reduce partition columns: _col4 (type: string), _col3 (type: string) + Map-reduce partition columns: _col3 (type: string), _col4 (type: string) Statistics: Num rows: 7 Data size: 924 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col13 (type: bigint), _col14 (type: bigint) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) Reducer 4 Execution mode: llap Reduce Operator Tree: @@ -2343,12 +2351,12 @@ STAGE PLANS: condition map: Left Outer Join 0 to 1 keys: - 0 _col4 (type: string), _col3 (type: string) - 1 _col1 (type: string), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col13, _col14, _col17 + 0 _col3 (type: string), _col4 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col14 Statistics: Num rows: 7 Data size: 1016 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col13 = 0)) THEN (false) WHEN (_col13 is null) THEN (false) WHEN (_col17 is not null) THEN (true) WHEN (_col3 is null) THEN (null) WHEN ((_col14 < _col13)) THEN (true) ELSE (false) END) (type: boolean) + predicate: (not CASE WHEN ((_col10 = 0)) THEN (false) WHEN (_col10 is null) THEN (false) WHEN (_col14 is not null) THEN (true) WHEN (_col3 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (true) ELSE (false) END) (type: boolean) Statistics: Num rows: 4 Data size: 580 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) @@ -2368,16 +2376,17 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 951 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) + Statistics: Num rows: 13 Data size: 4121 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string), _col2 (type: string) + mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 951 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 6 Data size: 1902 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) sort order: +++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string), _col2 (type: string) - Statistics: Num rows: 3 Data size: 951 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 6 Data size: 1902 Basic stats: COMPLETE Column stats: COMPLETE Reducer 8 Execution mode: llap Reduce Operator Tree: @@ -2433,32 +2442,32 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Input: default@part_null #### A masked pattern was here #### -17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve -15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu -40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s -191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle -78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr +86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously +191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle 105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ -112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car 146985 almond aquamarine midnight light salmon Manufacturer#2 Brand#23 MEDIUM BURNISHED COPPER 2 SM CASE 2031.98 s cajole caref +132666 almond aquamarine rose maroon antique Manufacturer#2 Brand#24 SMALL POLISHED NICKEL 25 MED BOX 1698.66 even +195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir +40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s +112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car +144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about 17273 almond antique forest lavender goldenrod Manufacturer#3 Brand#35 PROMO ANODIZED TIN 14 JUMBO CASE 1190.27 along the -110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously -121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h -121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h -48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i +17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick -155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra -45261 almond aquamarine floral ivory bisque Manufacturer#4 Brand#42 SMALL PLATED STEEL 27 WRAP CASE 1206.26 careful -132666 almond aquamarine rose maroon antique Manufacturer#2 Brand#24 SMALL POLISHED NICKEL 25 MED BOX 1698.66 even -86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful +48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i +45261 almond aquamarine floral ivory bisque Manufacturer#4 Brand#42 SMALL PLATED STEEL 27 WRAP CASE 1206.26 careful 42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl -195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de -144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about +78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith +192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir +15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu +155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra PREHOOK: query: explain select * from part_null where p_name IN (select p_name from part where part.p_type = part_null.p_type) AND p_brand NOT IN (select p_type from part where part.p_size = part_null.p_size) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_name IN (select p_name from part where part.p_type = part_null.p_type) AND p_brand NOT IN (select p_type from part where part.p_size = part_null.p_size) @@ -2472,7 +2481,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) @@ -2485,16 +2494,19 @@ STAGE PLANS: TableScan alias: part_null Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Filter Operator + predicate: (p_name is not null and p_type is not null) (type: boolean) Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string), _col4 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col4 (type: string) + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Reduce Output Operator + key expressions: _col1 (type: string), _col4 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col4 (type: string) + Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs Map 5 @@ -2503,7 +2515,7 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: p_type is not null (type: boolean) + predicate: (p_name is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_name (type: string), p_type (type: string) @@ -2554,17 +2566,17 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Inner Join 0 to 1 + Left Semi Join 0 to 1 keys: 0 _col1 (type: string), _col4 (type: string) 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 14 Data size: 3217 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1485 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col5 (type: int) sort order: + Map-reduce partition columns: _col5 (type: int) - Statistics: Num rows: 14 Data size: 3217 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 1485 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 3 Execution mode: llap @@ -2575,14 +2587,14 @@ STAGE PLANS: keys: 0 _col5 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13 - Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11 + Statistics: Num rows: 14 Data size: 286 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: string), _col5 (type: int) sort order: ++ Map-reduce partition columns: _col3 (type: string), _col5 (type: int) - Statistics: Num rows: 15 Data size: 3538 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col12 (type: bigint), _col13 (type: bigint) + Statistics: Num rows: 14 Data size: 286 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col10 (type: bigint), _col11 (type: bigint) Reducer 4 Execution mode: llap Reduce Operator Tree: @@ -2592,18 +2604,18 @@ STAGE PLANS: keys: 0 _col3 (type: string), _col5 (type: int) 1 _col0 (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col12, _col13, _col16 - Statistics: Num rows: 16 Data size: 3891 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col14 + Statistics: Num rows: 15 Data size: 314 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col12 = 0)) THEN (false) WHEN (_col12 is null) THEN (false) WHEN (_col16 is not null) THEN (true) WHEN (_col3 is null) THEN (null) WHEN ((_col13 < _col12)) THEN (true) ELSE (false) END) (type: boolean) - Statistics: Num rows: 8 Data size: 1945 Basic stats: COMPLETE Column stats: NONE + predicate: (not CASE WHEN ((_col10 = 0)) THEN (false) WHEN (_col10 is null) THEN (false) WHEN (_col14 is not null) THEN (true) WHEN (_col3 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (true) ELSE (false) END) (type: boolean) + Statistics: Num rows: 8 Data size: 167 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 8 Data size: 1945 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 167 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 1945 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 167 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2616,11 +2628,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 2925 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2925 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 1350 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 6 Data size: 1350 Basic stats: COMPLETE Column stats: COMPLETE Reducer 8 Execution mode: llap Reduce Operator Tree: @@ -2713,9 +2730,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2723,16 +2740,19 @@ STAGE PLANS: TableScan alias: part_null Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Filter Operator + predicate: (p_name is not null and p_type is not null) (type: boolean) Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string), _col4 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col4 (type: string) + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Reduce Output Operator + key expressions: _col1 (type: string), _col4 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col4 (type: string) + Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs Map 3 @@ -2741,7 +2761,7 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: p_type is not null (type: boolean) + predicate: (p_brand is not null and p_name is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_name (type: string), p_brand (type: string), p_type (type: string) @@ -2755,24 +2775,28 @@ STAGE PLANS: value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Map 6 Map Operator Tree: TableScan alias: pp Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: p_type is not null (type: boolean) + predicate: (p_brand is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_brand (type: string) - mode: hash + Select Operator + expressions: p_type (type: string), p_brand (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -2785,10 +2809,10 @@ STAGE PLANS: 0 _col1 (type: string), _col4 (type: string) 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 7 Data size: 1732 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 742 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 1732 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 742 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2798,43 +2822,40 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Inner Join 0 to 1 + Left Semi Join 0 to 1 keys: 0 _col2 (type: string), _col1 (type: string) - 1 _col1 (type: string), _col0 (type: string) + 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col2 Statistics: Num rows: 14 Data size: 3150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col2 (type: string) + Group By Operator + keys: _col0 (type: string), _col2 (type: string) + mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 3150 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 1575 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 7 Data size: 1575 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 7 Data size: 1575 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Reducer 5 Execution mode: llap Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: string), _col0 (type: string) + Statistics: Num rows: 7 Data size: 1575 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 675 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 3 Data size: 675 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -2878,7 +2899,7 @@ POSTHOOK: Input: default@part_null 17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve 33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful 78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith -Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product +Warning: Shuffle Join MERGEJOIN[67][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 5' is a cross product PREHOOK: query: explain select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -2898,18 +2919,39 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE) - Reducer 11 <- Map 9 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 7 <- Map 6 (XPROD_EDGE), Reducer 10 (XPROD_EDGE) - Reducer 8 <- Reducer 11 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 10 <- Map 8 (CUSTOM_SIMPLE_EDGE) + Reducer 12 <- Map 11 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 12 (ONE_TO_ONE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (XPROD_EDGE), Reducer 9 (XPROD_EDGE) + Reducer 6 <- Reducer 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE) + Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan + alias: li + Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((l_linenumber = 1) and l_orderkey is not null and l_partkey is not null) (type: boolean) + Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), 1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), 1 (type: int) + Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: int) + Execution mode: llap + LLAP IO: no inputs + Map 11 + Map Operator Tree: + TableScan alias: lineitem Statistics: Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator @@ -2927,45 +2969,25 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 5 - Map Operator Tree: - TableScan - alias: li - Statistics: Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) - Execution mode: llap - LLAP IO: no inputs - Map 6 + Map 4 Map Operator Tree: TableScan alias: lineitem Statistics: Num rows: 100 Data size: 10400 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((l_linenumber = l_linenumber) and (l_shipmode = 'AIR')) (type: boolean) - Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((l_linenumber = 1) and (l_shipmode = 'AIR') and l_orderkey is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: l_orderkey (type: int), l_linenumber (type: int), l_quantity (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 721 Basic stats: COMPLETE Column stats: COMPLETE + expressions: l_orderkey (type: int), l_quantity (type: double) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 2 Data size: 206 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 7 Data size: 721 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: double) + Statistics: Num rows: 2 Data size: 206 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: double) Execution mode: llap LLAP IO: no inputs - Map 9 + Map 8 Map Operator Tree: TableScan alias: lineitem @@ -2997,23 +3019,6 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(_col0) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: avg(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: double), true (type: boolean) outputColumnNames: _col0, _col1 @@ -3024,7 +3029,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) - Reducer 2 + Reducer 12 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3037,46 +3042,46 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 3 + Reducer 2 Execution mode: llap Reduce Operator Tree: Merge Join Operator condition map: - Inner Join 0 to 1 + Left Semi Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col0, _col1, _col3, _col4 - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + 0 _col0 (type: int), 1 (type: int) + 1 _col0 (type: int), _col1 (type: int) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: int), _col4 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: int), _col4 (type: int) - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col3 (type: int) - Reducer 4 + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: int) + Reducer 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator condition map: - Left Semi Join 0 to 1 + Inner Join 0 to 1 keys: - 0 _col1 (type: int), _col4 (type: int) - 1 _col0 (type: int), _col1 (type: int) - outputColumnNames: _col0, _col3 - Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col2, _col4 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col3 (type: int) + expressions: _col4 (type: int), _col2 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 7 + Reducer 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3085,15 +3090,15 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col4, _col5 - Statistics: Num rows: 7 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col2, _col4, _col5 + Statistics: Num rows: 2 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: double) sort order: + Map-reduce partition columns: _col2 (type: double) - Statistics: Num rows: 7 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: int), _col4 (type: bigint), _col5 (type: bigint) - Reducer 8 + Statistics: Num rows: 2 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col4 (type: bigint), _col5 (type: bigint) + Reducer 6 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3102,25 +3107,64 @@ STAGE PLANS: keys: 0 _col2 (type: double) 1 _col0 (type: double) - outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col7 - Statistics: Num rows: 7 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col2, _col4, _col5, _col7 + Statistics: Num rows: 2 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col4 = 0)) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col2 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) (type: boolean) - Statistics: Num rows: 4 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((_col4 = 0) or (_col7 is null and _col2 is not null and (_col5 >= _col4))) (type: boolean) + Statistics: Num rows: 2 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: int), _col1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: _col0 (type: int), _col1 (type: int) + keys: _col0 (type: int) mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), 1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: avg(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(_col0) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -3128,7 +3172,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 7' is a cross product +Warning: Shuffle Join MERGEJOIN[67][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 5' is a cross product PREHOOK: query: select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -3171,8 +3215,9 @@ STAGE PLANS: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) + Reducer 5 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Reducer 9 <- Map 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3180,17 +3225,20 @@ STAGE PLANS: TableScan alias: b Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (key > '9') (type: boolean) + predicate: ((key > '9') and value is not null) (type: boolean) Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) @@ -3210,7 +3258,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: value is not null (type: boolean) + predicate: (key is not null and value is not null) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: key (type: string), value (type: string) @@ -3236,20 +3284,16 @@ STAGE PLANS: Filter Operator predicate: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: value (type: string) + Group By Operator + keys: value (type: string) + mode: hash outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -3303,10 +3347,10 @@ STAGE PLANS: 0 _col2 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 33 Data size: 6138 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 2976 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 33 Data size: 6138 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 2976 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3321,18 +3365,18 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 134 Data size: 11658 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 67 Data size: 5829 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 67 Data size: 6365 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 33 Data size: 3135 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 67 Data size: 6365 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 33 Data size: 3135 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Reducer 6 Execution mode: llap @@ -3342,28 +3386,49 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 67 Data size: 6365 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 33 Data size: 3135 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: bigint) outputColumnNames: _col1 - Statistics: Num rows: 67 Data size: 536 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 33 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 67 Data size: 536 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 33 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 67 Data size: 536 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 33 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: bigint) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 33 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: bigint) sort order: + Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 33 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 125 Data size: 11375 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 125 Data size: 11375 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -3686,7 +3751,7 @@ POSTHOOK: Input: default@src 409 val_409 3 119 val_119 3 35 val_35 3 -Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from part where p_name IN (select p_name from part p where part.p_type <> '1') PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_name IN (select p_name from part p where part.p_type <> '1') @@ -3700,9 +3765,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (XPROD_EDGE), Reducer 6 (XPROD_EDGE) - Reducer 6 <- Map 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (XPROD_EDGE), Reducer 7 (XPROD_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3710,16 +3776,19 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Filter Operator + predicate: ((p_type <> '1') and p_name is not null) (type: boolean) Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col4 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col4 (type: string) + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Reduce Output Operator + key expressions: _col1 (type: string), _col4 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col4 (type: string) + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs Map 3 @@ -3727,17 +3796,20 @@ STAGE PLANS: TableScan alias: p Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: p_name is not null (type: boolean) Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 5 + Map 6 Map Operator Tree: TableScan alias: part @@ -3796,7 +3868,25 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 169 Data size: 38025 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 169 Data size: 38025 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 84 Data size: 18900 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 84 Data size: 18900 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 7 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3815,7 +3905,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[28][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select * from part where p_name IN (select p_name from part p where part.p_type <> '1') PREHOOK: type: QUERY PREHOOK: Input: default@part diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 8e2ca937af..2dd65b44be 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -341,23 +341,23 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: int) Filter Operator - predicate: (p_mfgr = p_mfgr) (type: boolean) - Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE + predicate: p_mfgr is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: p_mfgr (type: string), p_size (type: int) sort order: ++ Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: p_name (type: string) Filter Operator - predicate: (p_mfgr = p_mfgr) (type: boolean) - Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE + predicate: p_mfgr is not null (type: boolean) + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: p_mfgr (type: string), p_size (type: int) sort order: ++ Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: p_name (type: string) Execution mode: llap @@ -372,12 +372,12 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col4, _col5 - Statistics: Num rows: 26 Data size: 5894 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5974 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 26 Data size: 5894 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5974 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: int), _col4 (type: bigint), _col5 (type: bigint) Reducer 3 Execution mode: llap @@ -389,10 +389,10 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col8 - Statistics: Num rows: 26 Data size: 5906 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5994 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (not CASE WHEN ((_col4 = 0)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) (type: boolean) - Statistics: Num rows: 13 Data size: 2955 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 3007 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 @@ -410,7 +410,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) outputColumnNames: _col1, _col2, _col5 - Statistics: Num rows: 13 Data size: 6383 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -431,25 +431,25 @@ STAGE PLANS: window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 13 Data size: 6383 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (rank_window_0 <= 2) (type: boolean) - Statistics: Num rows: 4 Data size: 1964 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 3928 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 1964 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 3928 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col0) keys: _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 5 Execution mode: llap @@ -459,12 +459,12 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 6 Execution mode: llap @@ -472,7 +472,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) outputColumnNames: _col1, _col2, _col5 - Statistics: Num rows: 13 Data size: 6383 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -493,24 +493,24 @@ STAGE PLANS: window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 13 Data size: 6383 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (rank_window_0 <= 2) (type: boolean) - Statistics: Num rows: 4 Data size: 1964 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 3928 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 1964 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 3928 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE Reducer 7 Execution mode: llap Reduce Operator Tree: @@ -518,19 +518,19 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) Stage: Stage-0 @@ -925,25 +925,25 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (p_mfgr = p_mfgr) (type: boolean) - Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE + predicate: p_mfgr is not null (type: boolean) + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: p_mfgr (type: string), p_size (type: int) sort order: ++ Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 Reduce Output Operator key expressions: p_mfgr (type: string), p_size (type: int) sort order: ++ Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 Reduce Output Operator key expressions: p_mfgr (type: string), p_size (type: int) sort order: ++ Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs @@ -953,7 +953,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) outputColumnNames: _col2, _col5 - Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -974,25 +974,25 @@ STAGE PLANS: window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (rank_window_0 <= 2) (type: boolean) - Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: string), _col5 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) Reducer 11 Execution mode: llap @@ -1002,23 +1002,23 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), true (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string), _col0 (type: int) sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) - Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) Reducer 2 Execution mode: llap @@ -1030,19 +1030,19 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col4 - Statistics: Num rows: 26 Data size: 5846 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5886 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (sq_count_check(_col4, true) > 0) (type: boolean) - Statistics: Num rows: 8 Data size: 1808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1816 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 1808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1816 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 8 Data size: 1808 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: int) Reducer 3 Execution mode: llap @@ -1071,10 +1071,10 @@ STAGE PLANS: 0 _col1 (type: string), _col2 (type: int) 1 _col1 (type: string), _col0 (type: int) outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col10 - Statistics: Num rows: 8 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1924 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (not CASE WHEN ((_col6 = 0)) THEN (false) WHEN (_col6 is null) THEN (false) WHEN (_col10 is not null) THEN (true) WHEN (_col2 is null) THEN (null) WHEN ((_col7 < _col6)) THEN (true) ELSE (false) END) (type: boolean) - Statistics: Num rows: 4 Data size: 960 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 964 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 @@ -1092,7 +1092,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) outputColumnNames: _col2, _col5 - Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -1113,24 +1113,24 @@ STAGE PLANS: window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (rank_window_0 <= 2) (type: boolean) - Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE Reducer 7 Execution mode: llap Reduce Operator Tree: @@ -1138,18 +1138,18 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Reducer 8 Execution mode: llap @@ -1157,7 +1157,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) outputColumnNames: _col2, _col5 - Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -1178,25 +1178,25 @@ STAGE PLANS: window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (rank_window_0 <= 2) (type: boolean) - Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: string), _col5 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) Reducer 9 Execution mode: llap @@ -1206,18 +1206,18 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col1) keys: _col0 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Stage: Stage-0 @@ -3017,30 +3017,30 @@ STAGE PLANS: alias: p Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((p_partkey = p_partkey) and (p_size = p_size)) (type: boolean) - Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (p_partkey is not null and p_size is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(p_name) keys: p_partkey (type: int), p_size (type: int) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Group By Operator keys: p_partkey (type: int), p_name (type: string), p_size (type: int) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) sort order: +++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int) - Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -3053,12 +3053,12 @@ STAGE PLANS: 0 _col0 (type: int), _col5 (type: int) 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12 - Statistics: Num rows: 26 Data size: 16158 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 16318 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col5 (type: int) - Statistics: Num rows: 26 Data size: 16158 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 16318 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: bigint), _col12 (type: bigint) Reducer 3 Execution mode: llap @@ -3070,10 +3070,10 @@ STAGE PLANS: 0 _col0 (type: int), _col1 (type: string), _col5 (type: int) 1 _col1 (type: int), _col0 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col16 - Statistics: Num rows: 26 Data size: 16174 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 16374 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (not CASE WHEN ((_col11 = 0)) THEN (false) WHEN (_col11 is null) THEN (false) WHEN (_col16 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col12 < _col11)) THEN (true) ELSE (false) END) (type: boolean) - Statistics: Num rows: 13 Data size: 8087 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 8187 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 @@ -3093,12 +3093,12 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Reducer 6 Execution mode: llap @@ -3107,23 +3107,23 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: int), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), true (type: boolean) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 399 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1729 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int), _col0 (type: string), _col2 (type: int) sort order: +++ Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col2 (type: int) - Statistics: Num rows: 3 Data size: 399 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1729 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) Stage: Stage-0 diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out index c89d053b4a..342a35b10a 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out @@ -2290,19 +2290,19 @@ STAGE PLANS: alias: p Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((p_partkey = p_partkey) and (p_size = p_size)) (type: boolean) - Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (p_partkey is not null and p_size is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(p_name) keys: p_partkey (type: int), p_size (type: int) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs @@ -2316,10 +2316,10 @@ STAGE PLANS: 0 _col0 (type: int), _col5 (type: int) 1 _col2 (type: int), _col3 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 26 Data size: 16142 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 16262 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (UDFToLong(_col5) <> CASE WHEN (_col10 is null) THEN (0) ELSE (_col9) END) (type: boolean) - Statistics: Num rows: 26 Data size: 16142 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 16262 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 @@ -2339,16 +2339,16 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: bigint), true (type: boolean), _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 260 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int), _col3 (type: int) sort order: ++ Map-reduce partition columns: _col2 (type: int), _col3 (type: int) - Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 260 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: boolean) Stage: Stage-0 @@ -3851,12 +3851,12 @@ STAGE PLANS: alias: lineitem Statistics: Num rows: 100 Data size: 9600 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((l_linenumber = l_linenumber) and (l_shipmode = 'AIR')) (type: boolean) - Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean) + Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: l_orderkey (type: int), l_linenumber (type: int) outputColumnNames: l_orderkey, l_linenumber - Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(l_orderkey) keys: l_linenumber (type: int) @@ -4060,12 +4060,12 @@ STAGE PLANS: alias: lineitem Statistics: Num rows: 100 Data size: 9600 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((l_linenumber = l_linenumber) and (l_shipmode = 'AIR')) (type: boolean) - Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean) + Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: l_orderkey (type: int), l_linenumber (type: int) outputColumnNames: l_orderkey, l_linenumber - Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(l_orderkey) keys: l_linenumber (type: int) diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out b/ql/src/test/results/clientpositive/llap/subquery_select.q.out index 118f6ebccf..a085909967 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out @@ -1825,15 +1825,15 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((key > '9') and (value = value)) (type: boolean) - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: value (type: string) outputColumnNames: _col1 - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs @@ -1867,10 +1867,10 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 66 Data size: 11748 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 134 Data size: 23852 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 66 Data size: 11748 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 134 Data size: 23852 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1885,17 +1885,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col1, _col2 - Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 166 Data size: 45650 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col2 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 41 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE Reducer 4 Execution mode: llap Reduce Operator Tree: @@ -1903,12 +1903,12 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 41 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE Reducer 6 Execution mode: llap Reduce Operator Tree: @@ -1998,17 +1998,17 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((key > '9') and (value = value)) (type: boolean) - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: value (type: string) outputColumnNames: _col1 - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: value is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE @@ -2036,10 +2036,10 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 66 Data size: 11748 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 134 Data size: 23852 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 66 Data size: 11748 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 134 Data size: 23852 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2054,21 +2054,21 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col2 (type: string) outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 83 Data size: 20337 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 166 Data size: 40486 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: CASE WHEN (_col3 is null) THEN (null) ELSE (_col2) END (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 166 Data size: 45650 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 41 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE Reducer 4 Execution mode: llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/subquery_views.q.out b/ql/src/test/results/clientpositive/llap/subquery_views.q.out index a9a81133b5..af695691a7 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -157,33 +157,33 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((key = key) and (value = value) and (value > 'val_11')) (type: boolean) - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((value > 'val_11') and key is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(key) keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Filter Operator - predicate: ((key = key) and (value = value) and (value > 'val_11')) (type: boolean) - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((value > 'val_11') and key is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (key < '11') (type: boolean) Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE @@ -197,33 +197,33 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((key = key) and (value = value) and (value > 'val_11')) (type: boolean) - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((value > 'val_11') and key is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(key) keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Filter Operator - predicate: ((key = key) and (value = value) and (value > 'val_11')) (type: boolean) - Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((value > 'val_11') and key is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Map 11 @@ -260,12 +260,12 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col2, _col3, _col4 - Statistics: Num rows: 16 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col4 (type: string), _col2 (type: string) sort order: ++ Map-reduce partition columns: _col4 (type: string), _col2 (type: string) - Statistics: Num rows: 16 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) Reducer 12 Execution mode: llap @@ -305,19 +305,19 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) 2 _col4 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col4, _col5, _col9 - Statistics: Num rows: 19 Data size: 3762 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 87 Data size: 17226 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col9 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) - Statistics: Num rows: 9 Data size: 1782 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 43 Data size: 8514 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 43 Data size: 7654 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 9 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 43 Data size: 7654 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Reducer 3 Execution mode: llap @@ -329,10 +329,10 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 43 Data size: 7654 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 43 Data size: 7654 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -345,12 +345,12 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Reducer 5 Execution mode: llap @@ -359,16 +359,16 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string), _col3 (type: boolean) Reducer 6 Execution mode: llap @@ -380,12 +380,12 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col2, _col3, _col4 - Statistics: Num rows: 16 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col4 (type: string), _col2 (type: string) sort order: ++ Map-reduce partition columns: _col4 (type: string), _col2 (type: string) - Statistics: Num rows: 16 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) Reducer 7 Execution mode: llap @@ -399,24 +399,24 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) 2 _col4 (type: string), _col2 (type: string) outputColumnNames: _col0, _col4, _col5, _col9 - Statistics: Num rows: 19 Data size: 2033 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 87 Data size: 9309 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: CASE WHEN ((_col4 = 0)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col9 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) - Statistics: Num rows: 9 Data size: 963 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 43 Data size: 4601 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 9 Data size: 783 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 43 Data size: 3741 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 696 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 696 Basic stats: COMPLETE Column stats: COMPLETE Reducer 8 Execution mode: llap Reduce Operator Tree: @@ -425,12 +425,12 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Reducer 9 Execution mode: llap @@ -439,16 +439,16 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string), _col3 (type: boolean) Stage: Stage-0 diff --git a/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out index df104e9f3d..2f5a174a84 100644 --- a/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out @@ -402,9 +402,9 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualStringScalar(col 14, val AIR) -> boolean, FilterLongColEqualLongColumn(col 3, col 3) -> boolean) -> boolean - predicate: ((l_linenumber = l_linenumber) and (l_shipmode = 'AIR')) (type: boolean) - Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE + predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualStringScalar(col 14, val AIR) -> boolean, SelectColumnIsNotNull(col 3) -> boolean) -> boolean + predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean) + Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: l_orderkey (type: int), l_linenumber (type: int) outputColumnNames: _col0, _col1 @@ -412,7 +412,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0, 3] - Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 14 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator Group By Vectorization: className: VectorGroupByOperator @@ -425,7 +425,7 @@ STAGE PLANS: keys: _col0 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ @@ -434,7 +434,7 @@ STAGE PLANS: className: VectorReduceSinkMultiKeyOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/perf/spark/query16.q.out b/ql/src/test/results/clientpositive/perf/spark/query16.q.out index b74d721d41..205173a93a 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query16.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query16.q.out @@ -1,4 +1,3 @@ -Warning: Shuffle Join JOIN[33][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Work 'Reducer 17' is a cross product PREHOOK: query: explain select count(distinct cs_order_number) as `order count` @@ -70,7 +69,7 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 10 + Map 9 Map Operator Tree: TableScan alias: call_center @@ -93,7 +92,7 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 8 + Map 7 Map Operator Tree: TableScan alias: date_dim @@ -115,17 +114,13 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 12 <- Map 11 (GROUP, 24) - Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 1009), Reducer 18 (PARTITION-LEVEL SORT, 1009) - Reducer 15 <- Reducer 14 (GROUP, 1009) - Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 1), Map 19 (PARTITION-LEVEL SORT, 1), Map 20 (PARTITION-LEVEL SORT, 1), Map 21 (PARTITION-LEVEL SORT, 1) - Reducer 18 <- Reducer 17 (GROUP, 1009) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 494), Map 9 (PARTITION-LEVEL SORT, 494) - Reducer 3 <- Reducer 12 (PARTITION-LEVEL SORT, 418), Reducer 2 (PARTITION-LEVEL SORT, 418) - Reducer 4 <- Reducer 15 (PARTITION-LEVEL SORT, 1009), Reducer 3 (PARTITION-LEVEL SORT, 1009) - Reducer 5 <- Reducer 4 (GROUP, 1009) - Reducer 6 <- Reducer 5 (GROUP, 1) - Reducer 7 <- Reducer 6 (SORT, 1) + Reducer 11 <- Map 10 (GROUP, 24) + Reducer 13 <- Map 12 (GROUP, 305) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 494), Map 8 (PARTITION-LEVEL SORT, 494) + Reducer 3 <- Reducer 11 (PARTITION-LEVEL SORT, 570), Reducer 13 (PARTITION-LEVEL SORT, 570), Reducer 2 (PARTITION-LEVEL SORT, 570) + Reducer 4 <- Reducer 3 (GROUP, 447) + Reducer 5 <- Reducer 4 (GROUP, 1) + Reducer 6 <- Reducer 5 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -148,7 +143,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 8 + 1 Map 7 Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) @@ -158,7 +153,7 @@ STAGE PLANS: value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)) Local Work: Map Reduce Local Work - Map 11 + Map 10 Map Operator Tree: TableScan alias: cr1 @@ -176,65 +171,25 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE - Map 13 + Map 12 Map Operator Tree: TableScan alias: cs2 Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cs_warehouse_sk (type: int), cs_order_number (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Map 16 - Map Operator Tree: - TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 73049 Data size: 292196 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 73049 Data size: 292196 Basic stats: COMPLETE Column stats: COMPLETE - Map 19 - Map Operator Tree: - TableScan - alias: call_center - Statistics: Num rows: 60 Data size: 122700 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 60 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 60 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE - Map 20 - Map Operator Tree: - TableScan - alias: customer_address - Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 40000000 Data size: 160000000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 40000000 Data size: 160000000 Basic stats: COMPLETE Column stats: COMPLETE - Map 21 - Map Operator Tree: - TableScan - alias: cs1 - Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: cs_warehouse_sk (type: int), cs_order_number (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (cs_order_number is not null and cs_warehouse_sk is not null) (type: boolean) Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Group By Operator + keys: cs_order_number (type: int), cs_warehouse_sk (type: int) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int) - Map 9 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE + Map 8 Map Operator Tree: TableScan alias: customer_address @@ -251,7 +206,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 20000000 Data size: 20297597642 Basic stats: COMPLETE Column stats: NONE - Reducer 12 + Reducer 11 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) @@ -268,85 +223,22 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 14399440 Data size: 1528617286 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) - Reducer 14 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 5072854730221289472 Data size: 5072854730221289472 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col2 <> _col0) (type: boolean) - Statistics: Num rows: 5072854730221289472 Data size: 5072854730221289472 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: int), _col3 (type: int) - outputColumnNames: _col2, _col3 - Statistics: Num rows: 5072854730221289472 Data size: 5072854730221289472 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col2 (type: int), _col3 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5072854730221289472 Data size: 5072854730221289472 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 5072854730221289472 Data size: 5072854730221289472 Basic stats: COMPLETE Column stats: NONE - Reducer 15 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2536427365110644736 Data size: 2536427365110644736 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2536427365110644736 Data size: 2536427365110644736 Basic stats: COMPLETE Column stats: NONE - Reducer 17 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - Inner Join 0 to 3 - keys: - 0 - 1 - 2 - 3 - outputColumnNames: _col3, _col4 - Statistics: Num rows: 9223372036854775807 Data size: 9223372036854775807 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col4 (type: int), _col3 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 9223372036854775807 Data size: 9223372036854775807 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 9223372036854775807 Data size: 9223372036854775807 Basic stats: COMPLETE Column stats: NONE - Reducer 18 + Reducer 13 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 4611686018427387903 Data size: 4611686018427387904 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 143994918 Data size: 19499804476 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4611686018427387903 Data size: 4611686018427387904 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 143994918 Data size: 19499804476 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4611686018427387903 Data size: 4611686018427387904 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 143994918 Data size: 19499804476 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) Reducer 2 Local Work: @@ -368,7 +260,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col3, _col4, _col5, _col6 input vertices: - 1 Map 10 + 1 Map 9 Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col4 (type: int) @@ -381,58 +273,45 @@ STAGE PLANS: Join Operator condition map: Left Outer Join 0 to 1 + Inner Join 0 to 2 keys: 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col3, _col4, _col5, _col6, _col14 - Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: int), _col4 (type: int) - sort order: ++ - Map-reduce partition columns: _col3 (type: int), _col4 (type: int) - Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE - value expressions: _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col14 (type: boolean) - Reducer 4 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col3 (type: int), _col4 (type: int) - 1 _col0 (type: int), _col1 (type: int) - outputColumnNames: _col4, _col5, _col6, _col14 - Statistics: Num rows: 2790070162094850048 Data size: 2790070162094850048 Basic stats: COMPLETE Column stats: NONE + 2 _col1 (type: int) + outputColumnNames: _col3, _col4, _col5, _col6, _col14, _col15 + residual filter predicates: {(_col3 <> _col15)} + Statistics: Num rows: 843291907 Data size: 114198664830 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col14 (type: boolean) outputColumnNames: _col4, _col5, _col6, _col16 - Statistics: Num rows: 2790070162094850048 Data size: 2790070162094850048 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 843291907 Data size: 114198664830 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col16 is null (type: boolean) - Statistics: Num rows: 1395035081047425024 Data size: 1395035081047425024 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 421645953 Data size: 57099332347 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)) outputColumnNames: _col4, _col5, _col6 - Statistics: Num rows: 1395035081047425024 Data size: 1395035081047425024 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 421645953 Data size: 57099332347 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col5), sum(_col6) keys: _col4 (type: int) mode: hash outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 1395035081047425024 Data size: 1395035081047425024 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 421645953 Data size: 57099332347 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1395035081047425024 Data size: 1395035081047425024 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 421645953 Data size: 57099332347 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(17,2)), _col3 (type: decimal(17,2)) - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0), sum(VALUE._col1) keys: KEY._col0 (type: int) mode: partial2 outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1395035081047425024 Data size: 1395035081047425024 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 421645953 Data size: 57099332347 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col0), sum(_col1), sum(_col2) mode: partial2 @@ -442,7 +321,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 344 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: decimal(17,2)), _col2 (type: decimal(17,2)) - Reducer 6 + Reducer 5 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) @@ -459,7 +338,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 344 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(17,2)) - Reducer 7 + Reducer 6 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: bigint), VALUE._col0 (type: decimal(17,2)), VALUE._col1 (type: decimal(17,2)) diff --git a/ql/src/test/results/clientpositive/perf/spark/query94.q.out b/ql/src/test/results/clientpositive/perf/spark/query94.q.out index 43b8c77bdc..4ee2ed87e2 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query94.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query94.q.out @@ -1,4 +1,3 @@ -Warning: Shuffle Join JOIN[33][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Work 'Reducer 17' is a cross product PREHOOK: query: explain select count(distinct ws_order_number) as `order count` @@ -66,7 +65,7 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 10 + Map 9 Map Operator Tree: TableScan alias: web_site @@ -89,7 +88,7 @@ STAGE PLANS: Spark #### A masked pattern was here #### Vertices: - Map 8 + Map 7 Map Operator Tree: TableScan alias: date_dim @@ -111,17 +110,13 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 12 <- Map 11 (GROUP, 11) - Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 1009), Reducer 18 (PARTITION-LEVEL SORT, 1009) - Reducer 15 <- Reducer 14 (GROUP, 1009) - Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 1), Map 19 (PARTITION-LEVEL SORT, 1), Map 20 (PARTITION-LEVEL SORT, 1), Map 21 (PARTITION-LEVEL SORT, 1) - Reducer 18 <- Reducer 17 (GROUP, 1009) - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 327), Map 9 (PARTITION-LEVEL SORT, 327) - Reducer 3 <- Reducer 12 (PARTITION-LEVEL SORT, 209), Reducer 2 (PARTITION-LEVEL SORT, 209) - Reducer 4 <- Reducer 15 (PARTITION-LEVEL SORT, 1009), Reducer 3 (PARTITION-LEVEL SORT, 1009) - Reducer 5 <- Reducer 4 (GROUP, 1009) - Reducer 6 <- Reducer 5 (GROUP, 1) - Reducer 7 <- Reducer 6 (SORT, 1) + Reducer 11 <- Map 10 (GROUP, 11) + Reducer 13 <- Map 12 (GROUP, 153) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 327), Map 8 (PARTITION-LEVEL SORT, 327) + Reducer 3 <- Reducer 11 (PARTITION-LEVEL SORT, 286), Reducer 13 (PARTITION-LEVEL SORT, 286), Reducer 2 (PARTITION-LEVEL SORT, 286) + Reducer 4 <- Reducer 3 (GROUP, 224) + Reducer 5 <- Reducer 4 (GROUP, 1) + Reducer 6 <- Reducer 5 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -144,7 +139,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6 input vertices: - 1 Map 8 + 1 Map 7 Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) @@ -154,7 +149,7 @@ STAGE PLANS: value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)) Local Work: Map Reduce Local Work - Map 11 + Map 10 Map Operator Tree: TableScan alias: wr1 @@ -172,65 +167,25 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE - Map 13 + Map 12 Map Operator Tree: TableScan alias: ws2 Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ws_warehouse_sk (type: int), ws_order_number (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Map 16 - Map Operator Tree: - TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 73049 Data size: 292196 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 73049 Data size: 292196 Basic stats: COMPLETE Column stats: COMPLETE - Map 19 - Map Operator Tree: - TableScan - alias: web_site - Statistics: Num rows: 84 Data size: 155408 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 84 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 84 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE - Map 20 - Map Operator Tree: - TableScan - alias: customer_address - Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 40000000 Data size: 160000000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 40000000 Data size: 160000000 Basic stats: COMPLETE Column stats: COMPLETE - Map 21 - Map Operator Tree: - TableScan - alias: ws1 - Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: ws_warehouse_sk (type: int), ws_order_number (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: (ws_order_number is not null and ws_warehouse_sk is not null) (type: boolean) Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Group By Operator + keys: ws_order_number (type: int), ws_warehouse_sk (type: int) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int) - Map 9 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE + Map 8 Map Operator Tree: TableScan alias: customer_address @@ -247,7 +202,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 20000000 Data size: 20297597642 Basic stats: COMPLETE Column stats: NONE - Reducer 12 + Reducer 11 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) @@ -264,85 +219,22 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 7199233 Data size: 662597045 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) - Reducer 14 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 5072854730221289472 Data size: 5072854730221289472 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col2 <> _col0) (type: boolean) - Statistics: Num rows: 5072854730221289472 Data size: 5072854730221289472 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: int), _col3 (type: int) - outputColumnNames: _col2, _col3 - Statistics: Num rows: 5072854730221289472 Data size: 5072854730221289472 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col2 (type: int), _col3 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5072854730221289472 Data size: 5072854730221289472 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 5072854730221289472 Data size: 5072854730221289472 Basic stats: COMPLETE Column stats: NONE - Reducer 15 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2536427365110644736 Data size: 2536427365110644736 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 2536427365110644736 Data size: 2536427365110644736 Basic stats: COMPLETE Column stats: NONE - Reducer 17 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - Inner Join 0 to 3 - keys: - 0 - 1 - 2 - 3 - outputColumnNames: _col3, _col4 - Statistics: Num rows: 9223372036854775807 Data size: 9223372036854775807 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col4 (type: int), _col3 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 9223372036854775807 Data size: 9223372036854775807 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 9223372036854775807 Data size: 9223372036854775807 Basic stats: COMPLETE Column stats: NONE - Reducer 18 + Reducer 13 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 4611686018427387903 Data size: 4611686018427387904 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 72001334 Data size: 9790099106 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4611686018427387903 Data size: 4611686018427387904 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 72001334 Data size: 9790099106 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 4611686018427387903 Data size: 4611686018427387904 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 72001334 Data size: 9790099106 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) Reducer 2 Local Work: @@ -364,7 +256,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col3, _col4, _col5, _col6 input vertices: - 1 Map 10 + 1 Map 9 Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col4 (type: int) @@ -377,58 +269,45 @@ STAGE PLANS: Join Operator condition map: Left Outer Join 0 to 1 + Inner Join 0 to 2 keys: 0 _col4 (type: int) 1 _col0 (type: int) - outputColumnNames: _col3, _col4, _col5, _col6, _col14 - Statistics: Num rows: 210834322 Data size: 28667370686 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col3 (type: int), _col4 (type: int) - sort order: ++ - Map-reduce partition columns: _col3 (type: int), _col4 (type: int) - Statistics: Num rows: 210834322 Data size: 28667370686 Basic stats: COMPLETE Column stats: NONE - value expressions: _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col14 (type: boolean) - Reducer 4 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col3 (type: int), _col4 (type: int) - 1 _col0 (type: int), _col1 (type: int) - outputColumnNames: _col4, _col5, _col6, _col14 - Statistics: Num rows: 2790070162094850048 Data size: 2790070162094850048 Basic stats: COMPLETE Column stats: NONE + 2 _col1 (type: int) + outputColumnNames: _col3, _col4, _col5, _col6, _col14, _col15 + residual filter predicates: {(_col3 <> _col15)} + Statistics: Num rows: 421668645 Data size: 57334741373 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col14 (type: boolean) outputColumnNames: _col4, _col5, _col6, _col16 - Statistics: Num rows: 2790070162094850048 Data size: 2790070162094850048 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 421668645 Data size: 57334741373 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col16 is null (type: boolean) - Statistics: Num rows: 1395035081047425024 Data size: 1395035081047425024 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 210834322 Data size: 28667370618 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)) outputColumnNames: _col4, _col5, _col6 - Statistics: Num rows: 1395035081047425024 Data size: 1395035081047425024 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 210834322 Data size: 28667370618 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col5), sum(_col6) keys: _col4 (type: int) mode: hash outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 1395035081047425024 Data size: 1395035081047425024 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 210834322 Data size: 28667370618 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1395035081047425024 Data size: 1395035081047425024 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 210834322 Data size: 28667370618 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(17,2)), _col3 (type: decimal(17,2)) - Reducer 5 + Reducer 4 Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0), sum(VALUE._col1) keys: KEY._col0 (type: int) mode: partial2 outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1395035081047425024 Data size: 1395035081047425024 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 210834322 Data size: 28667370618 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col0), sum(_col1), sum(_col2) mode: partial2 @@ -438,7 +317,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 344 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: decimal(17,2)), _col2 (type: decimal(17,2)) - Reducer 6 + Reducer 5 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2) @@ -455,7 +334,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 344 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(17,2)) - Reducer 7 + Reducer 6 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: bigint), VALUE._col0 (type: decimal(17,2)), VALUE._col1 (type: decimal(17,2)) diff --git a/ql/src/test/results/clientpositive/perf/tez/query14.q.out b/ql/src/test/results/clientpositive/perf/tez/query14.q.out index b2a45f155a..dfdd53de00 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query14.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query14.q.out @@ -1,6 +1,6 @@ -Warning: Shuffle Join MERGEJOIN[890][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[891][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 12' is a cross product Warning: Shuffle Join MERGEJOIN[892][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 16' is a cross product +Warning: Shuffle Join MERGEJOIN[891][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 12' is a cross product +Warning: Shuffle Join MERGEJOIN[890][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product PREHOOK: query: explain with cross_items as (select i_item_sk ss_item_sk diff --git a/ql/src/test/results/clientpositive/perf/tez/query16.q.out b/ql/src/test/results/clientpositive/perf/tez/query16.q.out index a7b710d6e1..b2b990e8a5 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query16.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query16.q.out @@ -1,4 +1,3 @@ -Warning: Shuffle Join MERGEJOIN[113][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 18' is a cross product PREHOOK: query: explain select count(distinct cs_order_number) as `order count` @@ -62,180 +61,132 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 15 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Map 21 (CUSTOM_SIMPLE_EDGE), Map 22 (CUSTOM_SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE) -Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 14 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 17 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Reducer 13 <- Map 12 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) +Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Reducer 13 (ONE_TO_ONE_EDGE), Reducer 15 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) +Reducer 8 <- Reducer 7 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 - File Output Operator [FS_74] - Limit [LIM_72] (rows=1 width=344) + Reducer 8 + File Output Operator [FS_50] + Limit [LIM_48] (rows=1 width=344) Number of rows:100 - Select Operator [SEL_71] (rows=1 width=344) + Select Operator [SEL_47] (rows=1 width=344) Output:["_col0","_col1","_col2"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_70] - Select Operator [SEL_69] (rows=1 width=344) + <-Reducer 7 [SIMPLE_EDGE] + SHUFFLE [RS_46] + Select Operator [SEL_45] (rows=1 width=344) Output:["_col1","_col2","_col3"] - Group By Operator [GBY_112] (rows=1 width=344) + Group By Operator [GBY_78] (rows=1 width=344) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 7 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_111] - Group By Operator [GBY_110] (rows=1 width=344) + <-Reducer 6 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_77] + Group By Operator [GBY_76] (rows=1 width=344) Output:["_col0","_col1","_col2"],aggregations:["count(_col0)","sum(_col1)","sum(_col2)"] - Group By Operator [GBY_109] (rows=1395035081047425024 width=1) + Group By Operator [GBY_75] (rows=421645953 width=135) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_108] + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_74] PartitionCols:_col0 - Group By Operator [GBY_107] (rows=1395035081047425024 width=1) + Group By Operator [GBY_73] (rows=421645953 width=135) Output:["_col0","_col2","_col3"],aggregations:["sum(_col5)","sum(_col6)"],keys:_col4 - Select Operator [SEL_65] (rows=1395035081047425024 width=1) + Select Operator [SEL_41] (rows=421645953 width=135) Output:["_col4","_col5","_col6"] - Filter Operator [FIL_64] (rows=1395035081047425024 width=1) + Filter Operator [FIL_40] (rows=421645953 width=135) predicate:_col16 is null - Select Operator [SEL_63] (rows=2790070162094850048 width=1) + Select Operator [SEL_39] (rows=843291907 width=135) Output:["_col4","_col5","_col6","_col16"] - Merge Join Operator [MERGEJOIN_119] (rows=2790070162094850048 width=1) - Conds:RS_60._col3, _col4=RS_61._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"] - <-Reducer 17 [ONE_TO_ONE_EDGE] - FORWARD [RS_61] - PartitionCols:_col0, _col1 - Group By Operator [GBY_46] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0, _col1 - Group By Operator [GBY_44] (rows=5072854730221289472 width=1) - Output:["_col0","_col1"],keys:_col2, _col3 - Select Operator [SEL_43] (rows=5072854730221289472 width=1) - Output:["_col2","_col3"] - Filter Operator [FIL_42] (rows=5072854730221289472 width=1) - predicate:(_col2 <> _col0) - Merge Join Operator [MERGEJOIN_117] (rows=5072854730221289472 width=1) - Conds:RS_39._col1=RS_40._col1(Inner),Output:["_col0","_col2","_col3"] - <-Map 15 [SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_39] - PartitionCols:_col1 - Select Operator [SEL_20] (rows=287989836 width=135) - Output:["_col0","_col1"] - TableScan [TS_19] (rows=287989836 width=135) - default@catalog_sales,cs2,Tbl:COMPLETE,Col:NONE,Output:["cs_warehouse_sk","cs_order_number"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1 - Select Operator [SEL_38] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"] - Group By Operator [GBY_37] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=9223372036854775807 width=1) - Output:["_col0","_col1"],keys:_col4, _col3 - Merge Join Operator [MERGEJOIN_113] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"] - <-Map 15 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_32] - Please refer to the previous Select Operator [SEL_20] - <-Map 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_29] - Select Operator [SEL_22] (rows=73049 width=4) - TableScan [TS_21] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE - <-Map 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_30] - Select Operator [SEL_24] (rows=60 width=4) - TableScan [TS_23] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:COMPLETE - <-Map 22 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_31] - Select Operator [SEL_26] (rows=40000000 width=4) - TableScan [TS_25] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col3, _col4 - Merge Join Operator [MERGEJOIN_118] (rows=421645953 width=135) - Conds:RS_57._col4=RS_58._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"] - <-Reducer 14 [ONE_TO_ONE_EDGE] - FORWARD [RS_58] + Merge Join Operator [MERGEJOIN_82] (rows=843291907 width=135) + Conds:RS_35._col4=RS_36._col0(Left Outer),RS_35._col4=RS_37._col1(Inner),Output:["_col3","_col4","_col5","_col6","_col14","_col15"],residual filter predicates:{(_col3 <> _col15)} + <-Reducer 13 [ONE_TO_ONE_EDGE] + FORWARD [RS_36] + PartitionCols:_col0 + Select Operator [SEL_18] (rows=14399440 width=106) + Output:["_col0","_col1"] + Group By Operator [GBY_17] (rows=14399440 width=106) + Output:["_col0"],keys:KEY._col0 + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_16] + PartitionCols:_col0 + Group By Operator [GBY_15] (rows=28798881 width=106) + Output:["_col0"],keys:cr_order_number + Filter Operator [FIL_71] (rows=28798881 width=106) + predicate:cr_order_number is not null + TableScan [TS_12] (rows=28798881 width=106) + default@catalog_returns,cr1,Tbl:COMPLETE,Col:NONE,Output:["cr_order_number"] + <-Reducer 15 [SIMPLE_EDGE] + SHUFFLE [RS_37] + PartitionCols:_col1 + Select Operator [SEL_25] (rows=143994918 width=135) + Output:["_col0","_col1"] + Group By Operator [GBY_24] (rows=143994918 width=135) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Map 14 [SIMPLE_EDGE] + SHUFFLE [RS_23] + PartitionCols:_col0, _col1 + Group By Operator [GBY_22] (rows=287989836 width=135) + Output:["_col0","_col1"],keys:cs_order_number, cs_warehouse_sk + Filter Operator [FIL_72] (rows=287989836 width=135) + predicate:(cs_order_number is not null and cs_warehouse_sk is not null) + TableScan [TS_19] (rows=287989836 width=135) + default@catalog_sales,cs2,Tbl:COMPLETE,Col:NONE,Output:["cs_warehouse_sk","cs_order_number"] + <-Reducer 4 [SIMPLE_EDGE] + SHUFFLE [RS_35] + PartitionCols:_col4 + Merge Join Operator [MERGEJOIN_81] (rows=383314495 width=135) + Conds:RS_32._col2=RS_33._col0(Inner),Output:["_col3","_col4","_col5","_col6"] + <-Map 11 [SIMPLE_EDGE] + SHUFFLE [RS_33] PartitionCols:_col0 - Select Operator [SEL_18] (rows=14399440 width=106) - Output:["_col0","_col1"] - Group By Operator [GBY_17] (rows=14399440 width=106) - Output:["_col0"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=28798881 width=106) - Output:["_col0"],keys:cr_order_number - Filter Operator [FIL_104] (rows=28798881 width=106) - predicate:cr_order_number is not null - TableScan [TS_12] (rows=28798881 width=106) - default@catalog_returns,cr1,Tbl:COMPLETE,Col:NONE,Output:["cr_order_number"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_116] (rows=383314495 width=135) - Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_55] + Select Operator [SEL_11] (rows=30 width=2045) + Output:["_col0"] + Filter Operator [FIL_70] (rows=30 width=2045) + predicate:((cc_county) IN ('Ziebach County', 'Levy County', 'Huron County', 'Franklin Parish', 'Daviess County') and cc_call_center_sk is not null) + TableScan [TS_9] (rows=60 width=2045) + default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_county"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_32] + PartitionCols:_col2 + Merge Join Operator [MERGEJOIN_80] (rows=348467716 width=135) + Conds:RS_29._col1=RS_30._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"] + <-Map 10 [SIMPLE_EDGE] + SHUFFLE [RS_30] PartitionCols:_col0 - Select Operator [SEL_11] (rows=30 width=2045) + Select Operator [SEL_8] (rows=20000000 width=1014) Output:["_col0"] - Filter Operator [FIL_103] (rows=30 width=2045) - predicate:((cc_county) IN ('Ziebach County', 'Levy County', 'Huron County', 'Franklin Parish', 'Daviess County') and cc_call_center_sk is not null) - TableScan [TS_9] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_county"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_115] (rows=348467716 width=135) - Conds:RS_51._col1=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_52] + Filter Operator [FIL_69] (rows=20000000 width=1014) + predicate:((ca_state = 'NY') and ca_address_sk is not null) + TableScan [TS_6] (rows=40000000 width=1014) + default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_29] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_79] (rows=316788826 width=135) + Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_26] PartitionCols:_col0 - Select Operator [SEL_8] (rows=20000000 width=1014) + Select Operator [SEL_2] (rows=287989836 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Filter Operator [FIL_67] (rows=287989836 width=135) + predicate:(cs_call_center_sk is not null and cs_ship_addr_sk is not null and cs_ship_date_sk is not null) + TableScan [TS_0] (rows=287989836 width=135) + default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_ship_date_sk","cs_ship_addr_sk","cs_call_center_sk","cs_warehouse_sk","cs_order_number","cs_ext_ship_cost","cs_net_profit"] + <-Map 9 [SIMPLE_EDGE] + SHUFFLE [RS_27] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=8116 width=1119) Output:["_col0"] - Filter Operator [FIL_102] (rows=20000000 width=1014) - predicate:((ca_state = 'NY') and ca_address_sk is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_114] (rows=316788826 width=135) - Conds:RS_48._col0=RS_49._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_100] (rows=287989836 width=135) - predicate:(cs_call_center_sk is not null and cs_ship_addr_sk is not null and cs_ship_date_sk is not null) - TableScan [TS_0] (rows=287989836 width=135) - default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_ship_date_sk","cs_ship_addr_sk","cs_call_center_sk","cs_warehouse_sk","cs_order_number","cs_ext_ship_cost","cs_net_profit"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_101] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 2001-04-01 00:00:00.0 AND 2001-05-31 01:00:00.0 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Filter Operator [FIL_68] (rows=8116 width=1119) + predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 2001-04-01 00:00:00.0 AND 2001-05-31 01:00:00.0 and d_date_sk is not null) + TableScan [TS_3] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] diff --git a/ql/src/test/results/clientpositive/perf/tez/query23.q.out b/ql/src/test/results/clientpositive/perf/tez/query23.q.out index 7112de61d9..3507278931 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query23.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query23.q.out @@ -1,5 +1,5 @@ -Warning: Shuffle Join MERGEJOIN[367][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 25' is a cross product Warning: Shuffle Join MERGEJOIN[369][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 30' is a cross product +Warning: Shuffle Join MERGEJOIN[367][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 25' is a cross product PREHOOK: query: explain with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt diff --git a/ql/src/test/results/clientpositive/perf/tez/query94.q.out b/ql/src/test/results/clientpositive/perf/tez/query94.q.out index 7674aa7f7c..8a9e5ca263 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query94.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query94.q.out @@ -1,4 +1,3 @@ -Warning: Shuffle Join MERGEJOIN[113][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 18' is a cross product PREHOOK: query: explain select count(distinct ws_order_number) as `order count` @@ -58,180 +57,132 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 15 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Map 21 (CUSTOM_SIMPLE_EDGE), Map 22 (CUSTOM_SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE) -Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 14 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 17 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Reducer 13 <- Map 12 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) +Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Reducer 13 (ONE_TO_ONE_EDGE), Reducer 15 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) +Reducer 8 <- Reducer 7 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 - File Output Operator [FS_74] - Limit [LIM_72] (rows=1 width=344) + Reducer 8 + File Output Operator [FS_50] + Limit [LIM_48] (rows=1 width=344) Number of rows:100 - Select Operator [SEL_71] (rows=1 width=344) + Select Operator [SEL_47] (rows=1 width=344) Output:["_col0","_col1","_col2"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_70] - Select Operator [SEL_69] (rows=1 width=344) + <-Reducer 7 [SIMPLE_EDGE] + SHUFFLE [RS_46] + Select Operator [SEL_45] (rows=1 width=344) Output:["_col1","_col2","_col3"] - Group By Operator [GBY_112] (rows=1 width=344) + Group By Operator [GBY_78] (rows=1 width=344) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 7 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_111] - Group By Operator [GBY_110] (rows=1 width=344) + <-Reducer 6 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_77] + Group By Operator [GBY_76] (rows=1 width=344) Output:["_col0","_col1","_col2"],aggregations:["count(_col0)","sum(_col1)","sum(_col2)"] - Group By Operator [GBY_109] (rows=1395035081047425024 width=1) + Group By Operator [GBY_75] (rows=210834322 width=135) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_108] + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_74] PartitionCols:_col0 - Group By Operator [GBY_107] (rows=1395035081047425024 width=1) + Group By Operator [GBY_73] (rows=210834322 width=135) Output:["_col0","_col2","_col3"],aggregations:["sum(_col5)","sum(_col6)"],keys:_col4 - Select Operator [SEL_65] (rows=1395035081047425024 width=1) + Select Operator [SEL_41] (rows=210834322 width=135) Output:["_col4","_col5","_col6"] - Filter Operator [FIL_64] (rows=1395035081047425024 width=1) + Filter Operator [FIL_40] (rows=210834322 width=135) predicate:_col16 is null - Select Operator [SEL_63] (rows=2790070162094850048 width=1) + Select Operator [SEL_39] (rows=421668645 width=135) Output:["_col4","_col5","_col6","_col16"] - Merge Join Operator [MERGEJOIN_119] (rows=2790070162094850048 width=1) - Conds:RS_60._col3, _col4=RS_61._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"] - <-Reducer 17 [ONE_TO_ONE_EDGE] - FORWARD [RS_61] - PartitionCols:_col0, _col1 - Group By Operator [GBY_46] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0, _col1 - Group By Operator [GBY_44] (rows=5072854730221289472 width=1) - Output:["_col0","_col1"],keys:_col2, _col3 - Select Operator [SEL_43] (rows=5072854730221289472 width=1) - Output:["_col2","_col3"] - Filter Operator [FIL_42] (rows=5072854730221289472 width=1) - predicate:(_col2 <> _col0) - Merge Join Operator [MERGEJOIN_117] (rows=5072854730221289472 width=1) - Conds:RS_39._col1=RS_40._col1(Inner),Output:["_col0","_col2","_col3"] - <-Map 15 [SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_39] - PartitionCols:_col1 - Select Operator [SEL_20] (rows=144002668 width=135) - Output:["_col0","_col1"] - TableScan [TS_19] (rows=144002668 width=135) - default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1 - Select Operator [SEL_38] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"] - Group By Operator [GBY_37] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=9223372036854775807 width=1) - Output:["_col0","_col1"],keys:_col4, _col3 - Merge Join Operator [MERGEJOIN_113] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"] - <-Map 15 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_32] - Please refer to the previous Select Operator [SEL_20] - <-Map 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_29] - Select Operator [SEL_22] (rows=73049 width=4) - TableScan [TS_21] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE - <-Map 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_30] - Select Operator [SEL_24] (rows=84 width=4) - TableScan [TS_23] (rows=84 width=1850) - default@web_site,web_site,Tbl:COMPLETE,Col:COMPLETE - <-Map 22 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_31] - Select Operator [SEL_26] (rows=40000000 width=4) - TableScan [TS_25] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col3, _col4 - Merge Join Operator [MERGEJOIN_118] (rows=210834322 width=135) - Conds:RS_57._col4=RS_58._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"] - <-Reducer 14 [ONE_TO_ONE_EDGE] - FORWARD [RS_58] + Merge Join Operator [MERGEJOIN_82] (rows=421668645 width=135) + Conds:RS_35._col4=RS_36._col0(Left Outer),RS_35._col4=RS_37._col1(Inner),Output:["_col3","_col4","_col5","_col6","_col14","_col15"],residual filter predicates:{(_col3 <> _col15)} + <-Reducer 13 [ONE_TO_ONE_EDGE] + FORWARD [RS_36] + PartitionCols:_col0 + Select Operator [SEL_18] (rows=7199233 width=92) + Output:["_col0","_col1"] + Group By Operator [GBY_17] (rows=7199233 width=92) + Output:["_col0"],keys:KEY._col0 + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_16] + PartitionCols:_col0 + Group By Operator [GBY_15] (rows=14398467 width=92) + Output:["_col0"],keys:wr_order_number + Filter Operator [FIL_71] (rows=14398467 width=92) + predicate:wr_order_number is not null + TableScan [TS_12] (rows=14398467 width=92) + default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"] + <-Reducer 15 [SIMPLE_EDGE] + SHUFFLE [RS_37] + PartitionCols:_col1 + Select Operator [SEL_25] (rows=72001334 width=135) + Output:["_col0","_col1"] + Group By Operator [GBY_24] (rows=72001334 width=135) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Map 14 [SIMPLE_EDGE] + SHUFFLE [RS_23] + PartitionCols:_col0, _col1 + Group By Operator [GBY_22] (rows=144002668 width=135) + Output:["_col0","_col1"],keys:ws_order_number, ws_warehouse_sk + Filter Operator [FIL_72] (rows=144002668 width=135) + predicate:(ws_order_number is not null and ws_warehouse_sk is not null) + TableScan [TS_19] (rows=144002668 width=135) + default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] + <-Reducer 4 [SIMPLE_EDGE] + SHUFFLE [RS_35] + PartitionCols:_col4 + Merge Join Operator [MERGEJOIN_81] (rows=191667562 width=135) + Conds:RS_32._col2=RS_33._col0(Inner),Output:["_col3","_col4","_col5","_col6"] + <-Map 11 [SIMPLE_EDGE] + SHUFFLE [RS_33] PartitionCols:_col0 - Select Operator [SEL_18] (rows=7199233 width=92) - Output:["_col0","_col1"] - Group By Operator [GBY_17] (rows=7199233 width=92) - Output:["_col0"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Group By Operator [GBY_15] (rows=14398467 width=92) - Output:["_col0"],keys:wr_order_number - Filter Operator [FIL_104] (rows=14398467 width=92) - predicate:wr_order_number is not null - TableScan [TS_12] (rows=14398467 width=92) - default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_116] (rows=191667562 width=135) - Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_55] + Select Operator [SEL_11] (rows=42 width=1850) + Output:["_col0"] + Filter Operator [FIL_70] (rows=42 width=1850) + predicate:((web_company_name = 'pri') and web_site_sk is not null) + TableScan [TS_9] (rows=84 width=1850) + default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_32] + PartitionCols:_col2 + Merge Join Operator [MERGEJOIN_80] (rows=174243235 width=135) + Conds:RS_29._col1=RS_30._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"] + <-Map 10 [SIMPLE_EDGE] + SHUFFLE [RS_30] PartitionCols:_col0 - Select Operator [SEL_11] (rows=42 width=1850) + Select Operator [SEL_8] (rows=20000000 width=1014) Output:["_col0"] - Filter Operator [FIL_103] (rows=42 width=1850) - predicate:((web_company_name = 'pri') and web_site_sk is not null) - TableScan [TS_9] (rows=84 width=1850) - default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_115] (rows=174243235 width=135) - Conds:RS_51._col1=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_52] + Filter Operator [FIL_69] (rows=20000000 width=1014) + predicate:((ca_state = 'TX') and ca_address_sk is not null) + TableScan [TS_6] (rows=40000000 width=1014) + default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_29] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_79] (rows=158402938 width=135) + Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_26] PartitionCols:_col0 - Select Operator [SEL_8] (rows=20000000 width=1014) + Select Operator [SEL_2] (rows=144002668 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Filter Operator [FIL_67] (rows=144002668 width=135) + predicate:(ws_ship_addr_sk is not null and ws_ship_date_sk is not null and ws_web_site_sk is not null) + TableScan [TS_0] (rows=144002668 width=135) + default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_warehouse_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"] + <-Map 9 [SIMPLE_EDGE] + SHUFFLE [RS_27] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=8116 width=1119) Output:["_col0"] - Filter Operator [FIL_102] (rows=20000000 width=1014) - predicate:((ca_state = 'TX') and ca_address_sk is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_114] (rows=158402938 width=135) - Conds:RS_48._col0=RS_49._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_100] (rows=144002668 width=135) - predicate:(ws_ship_addr_sk is not null and ws_ship_date_sk is not null and ws_web_site_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_warehouse_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_101] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Filter Operator [FIL_68] (rows=8116 width=1119) + predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null) + TableScan [TS_3] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] diff --git a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out index b89f9f5905..5b57036d4a 100644 --- a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out +++ b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out @@ -122,22 +122,22 @@ STAGE PLANS: alias: lineitem Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((l_linenumber = l_linenumber) and (l_shipmode = 'AIR')) (type: boolean) - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean) + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: l_orderkey (type: int), l_linenumber (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator diff --git a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out index 76c74d9ab7..8ae90805e0 100644 --- a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out @@ -1693,11 +1693,11 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_19] - Select Operator [SEL_18] (rows=434 width=178) + Select Operator [SEL_18] (rows=366 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_17] (rows=434 width=178) + Filter Operator [FIL_17] (rows=366 width=179) predicate:_col3 is null - Join Operator [JOIN_16] (rows=500 width=178) + Join Operator [JOIN_16] (rows=500 width=179) Output:["_col0","_col1","_col3"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1","1":"_col0"} <-Map 1 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_14] @@ -1709,21 +1709,21 @@ Stage-0 <-Reducer 4 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_15] PartitionCols:_col0 - Select Operator [SEL_13] (rows=41 width=95) + Select Operator [SEL_13] (rows=83 width=95) Output:["_col0","_col1"] - Group By Operator [GBY_12] (rows=41 width=91) + Group By Operator [GBY_12] (rows=83 width=91) Output:["_col0"],keys:_col1 - Select Operator [SEL_8] (rows=41 width=178) + Select Operator [SEL_8] (rows=83 width=178) Output:["_col1"] - Group By Operator [GBY_7] (rows=41 width=178) + Group By Operator [GBY_7] (rows=83 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Map 3 [GROUP PARTITION-LEVEL SORT] GROUP PARTITION-LEVEL SORT [RS_6] PartitionCols:_col0 - Group By Operator [GBY_5] (rows=41 width=178) + Group By Operator [GBY_5] (rows=83 width=178) Output:["_col0","_col1"],keys:value, key - Filter Operator [FIL_20] (rows=83 width=178) - predicate:((value = value) and (value > 'val_2')) + Filter Operator [FIL_20] (rows=166 width=178) + predicate:(value > 'val_2') TableScan [TS_2] (rows=500 width=178) default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -1758,11 +1758,11 @@ Stage-0 Stage-1 Reducer 3 File Output Operator [FS_18] - Select Operator [SEL_17] (rows=234 width=178) + Select Operator [SEL_17] (rows=183 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=234 width=178) + Filter Operator [FIL_16] (rows=183 width=179) predicate:_col4 is null - Join Operator [JOIN_15] (rows=250 width=178) + Join Operator [JOIN_15] (rows=250 width=179) Output:["_col0","_col1","_col4"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"} <-Reducer 2 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_13] @@ -1781,17 +1781,17 @@ Stage-0 <-Reducer 5 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_14] PartitionCols:_col0, _col1 - Select Operator [SEL_12] (rows=20 width=182) + Select Operator [SEL_12] (rows=83 width=182) Output:["_col0","_col1","_col2"] - Group By Operator [GBY_11] (rows=20 width=178) + Group By Operator [GBY_11] (rows=83 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Map 4 [GROUP] GROUP [RS_10] PartitionCols:_col0, _col1 - Group By Operator [GBY_9] (rows=20 width=178) + Group By Operator [GBY_9] (rows=83 width=178) Output:["_col0","_col1"],keys:key, value - Filter Operator [FIL_19] (rows=41 width=178) - predicate:((key = key) and (value = value) and (value > 'val_12')) + Filter Operator [FIL_19] (rows=166 width=178) + predicate:((value > 'val_12') and key is not null) TableScan [TS_6] (rows=500 width=178) default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -1834,7 +1834,7 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_11] - Join Operator [JOIN_9] (rows=32 width=178) + Join Operator [JOIN_9] (rows=134 width=178) Output:["_col0","_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"} <-Map 1 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_7] @@ -1846,12 +1846,12 @@ Stage-0 <-Map 3 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_8] PartitionCols:_col0, _col1 - Group By Operator [GBY_6] (rows=20 width=178) + Group By Operator [GBY_6] (rows=83 width=178) Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_4] (rows=41 width=178) + Select Operator [SEL_4] (rows=166 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_12] (rows=41 width=178) - predicate:((key = key) and (value = value) and (value > 'val_9')) + Filter Operator [FIL_12] (rows=166 width=178) + predicate:((value > 'val_9') and key is not null) TableScan [TS_2] (rows=500 width=178) default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -1884,7 +1884,7 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_11] - Join Operator [JOIN_9] (rows=32 width=178) + Join Operator [JOIN_9] (rows=134 width=178) Output:["_col0","_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"} <-Map 1 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_7] @@ -1896,12 +1896,12 @@ Stage-0 <-Map 3 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_8] PartitionCols:_col0, _col1 - Group By Operator [GBY_6] (rows=20 width=178) + Group By Operator [GBY_6] (rows=83 width=178) Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_4] (rows=41 width=178) + Select Operator [SEL_4] (rows=166 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_12] (rows=41 width=178) - predicate:((key = key) and (value = value) and (value > 'val_9')) + Filter Operator [FIL_12] (rows=166 width=178) + predicate:((value > 'val_9') and key is not null) TableScan [TS_2] (rows=500 width=178) default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -1977,12 +1977,12 @@ Stage-0 <-Map 6 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_19] PartitionCols:_col0, _col1 - Group By Operator [GBY_17] (rows=2 width=8) + Group By Operator [GBY_17] (rows=4 width=8) Output:["_col0","_col1"],keys:_col0, _col1 - Select Operator [SEL_12] (rows=7 width=8) + Select Operator [SEL_12] (rows=14 width=8) Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=7 width=96) - predicate:((l_linenumber = l_linenumber) and (l_shipmode = 'AIR')) + Filter Operator [FIL_25] (rows=14 width=96) + predicate:((l_shipmode = 'AIR') and l_linenumber is not null) TableScan [TS_10] (rows=100 width=96) default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_linenumber","l_shipmode"] <-Reducer 3 [PARTITION-LEVEL SORT] @@ -2272,14 +2272,14 @@ Stage-0 File Output Operator [FS_26] Select Operator [SEL_25] (rows=13 width=223) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_24] (rows=13 width=227) + Filter Operator [FIL_24] (rows=13 width=231) predicate:(not CASE WHEN ((_col4 = 0)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) - Join Operator [JOIN_23] (rows=26 width=227) + Join Operator [JOIN_23] (rows=26 width=230) Output:["_col0","_col1","_col2","_col4","_col5","_col8"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"} <-Reducer 2 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_21] PartitionCols:_col0, _col1 - Join Operator [JOIN_20] (rows=26 width=226) + Join Operator [JOIN_20] (rows=26 width=229) Output:["_col0","_col1","_col2","_col4","_col5"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1","1":"_col0"} <-Map 1 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_18] @@ -2291,37 +2291,37 @@ Stage-0 <-Reducer 5 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_19] PartitionCols:_col0 - Group By Operator [GBY_7] (rows=1 width=114) + Group By Operator [GBY_7] (rows=2 width=114) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 <-Map 4 [GROUP] GROUP [RS_6] PartitionCols:_col0 - Group By Operator [GBY_5] (rows=1 width=114) + Group By Operator [GBY_5] (rows=2 width=114) Output:["_col0","_col1","_col2"],aggregations:["count()","count(p_name)"],keys:p_mfgr - Select Operator [SEL_4] (rows=4 width=223) + Select Operator [SEL_4] (rows=8 width=223) Output:["p_name","p_mfgr"] - Filter Operator [FIL_27] (rows=4 width=223) - predicate:((p_mfgr = p_mfgr) and (p_size < 10)) + Filter Operator [FIL_27] (rows=8 width=223) + predicate:((p_size < 10) and p_mfgr is not null) TableScan [TS_2] (rows=26 width=223) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] <-Reducer 7 [PARTITION-LEVEL SORT] PARTITION-LEVEL SORT [RS_22] PartitionCols:_col0, _col1 - Select Operator [SEL_17] (rows=2 width=223) + Select Operator [SEL_17] (rows=4 width=223) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_16] (rows=2 width=219) + Filter Operator [FIL_16] (rows=4 width=219) predicate:_col0 is not null - Group By Operator [GBY_14] (rows=2 width=219) + Group By Operator [GBY_14] (rows=4 width=219) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Map 6 [GROUP] GROUP [RS_13] PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=2 width=219) + Group By Operator [GBY_12] (rows=4 width=219) Output:["_col0","_col1"],keys:p_name, p_mfgr - Select Operator [SEL_11] (rows=4 width=223) + Select Operator [SEL_11] (rows=8 width=223) Output:["p_name","p_mfgr"] - Filter Operator [FIL_28] (rows=4 width=223) - predicate:((p_mfgr = p_mfgr) and (p_size < 10)) + Filter Operator [FIL_28] (rows=8 width=223) + predicate:((p_size < 10) and p_mfgr is not null) TableScan [TS_9] (rows=26 width=223) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] @@ -2457,7 +2457,7 @@ Stage-0 Select Operator [SEL_24] (rows=1 width=110) Output:["_col0","_col1"] Filter Operator [FIL_23] (rows=1 width=114) - predicate:(((_col2 - _col1) > 600.0) and (_col1 = _col1)) + predicate:(((_col2 - _col1) > 600.0) and _col1 is not null) Group By Operator [GBY_22] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 <-Map 6 [GROUP] @@ -2501,7 +2501,7 @@ Stage-0 Select Operator [SEL_12] (rows=1 width=114) Output:["_col0","_col1"] Filter Operator [FIL_11] (rows=1 width=114) - predicate:(((_col2 - _col1) > 600.0) and (_col1 = _col1)) + predicate:(((_col2 - _col1) > 600.0) and _col1 is not null) Group By Operator [GBY_10] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 <- Please refer to the previous Map 6 [GROUP] diff --git a/ql/src/test/results/clientpositive/spark/subquery_exists.q.out b/ql/src/test/results/clientpositive/spark/subquery_exists.q.out index dafe5b6d5b..2c861964ca 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_exists.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_exists.q.out @@ -47,22 +47,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = key) and (value = value) and (value > 'val_9')) (type: boolean) - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + predicate: ((value > 'val_9') and key is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -1035,383 +1035,3 @@ POSTHOOK: query: drop table t POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@t POSTHOOK: Output: default@t -PREHOOK: query: drop table if exists tx1 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table if exists tx1 -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table tx1 (a integer,b integer) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@tx1 -POSTHOOK: query: create table tx1 (a integer,b integer) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@tx1 -PREHOOK: query: insert into tx1 values (1, 1), - (1, 2), - (1, 3) -PREHOOK: type: QUERY -PREHOOK: Output: default@tx1 -POSTHOOK: query: insert into tx1 values (1, 1), - (1, 2), - (1, 3) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@tx1 -POSTHOOK: Lineage: tx1.a EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: tx1.b EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -PREHOOK: query: select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -PREHOOK: type: QUERY -PREHOOK: Input: default@tx1 -#### A masked pattern was here #### -POSTHOOK: query: select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@tx1 -#### A masked pattern was here #### -3 3 -PREHOOK: query: explain select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -PREHOOK: type: QUERY -POSTHOOK: query: explain select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Spark - Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2) - Reducer 3 <- Reducer 2 (GROUP, 1) - Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 2), Reducer 7 (PARTITION-LEVEL SORT, 2) - Reducer 7 <- Map 6 (GROUP, 2) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: u - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Map 4 - Map Operator Tree: - TableScan - alias: v - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) - Map 6 - Map Operator Tree: - TableScan - alias: u - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: a, b - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int), b (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 _col0 (type: int), _col1 (type: int) - 1 _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), 3 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col3 <> _col1) (type: boolean) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: int), _col3 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reducer 7 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: drop table tx1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@tx1 -PREHOOK: Output: default@tx1 -POSTHOOK: query: drop table tx1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@tx1 -POSTHOOK: Output: default@tx1 -PREHOOK: query: create table t1(i int, j int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t1 -POSTHOOK: query: create table t1(i int, j int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1 -PREHOOK: query: insert into t1 values(4,1) -PREHOOK: type: QUERY -PREHOOK: Output: default@t1 -POSTHOOK: query: insert into t1 values(4,1) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@t1 -POSTHOOK: Lineage: t1.i EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: t1.j EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -PREHOOK: query: create table t2(i int, j int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t2 -POSTHOOK: query: create table t2(i int, j int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t2 -PREHOOK: query: insert into t2 values(4,2),(4,3),(4,5) -PREHOOK: type: QUERY -PREHOOK: Output: default@t2 -POSTHOOK: query: insert into t2 values(4,2),(4,3),(4,5) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@t2 -POSTHOOK: Lineage: t2.i EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: t2.j EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product -PREHOOK: query: explain select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Spark - Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2) - Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 1), Reducer 6 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (GROUP, 2) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Map 3 - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int) - Map 5 - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: j (type: int) - outputColumnNames: j - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reducer 2 - Reduce Operator Tree: - Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 _col0 (type: int), _col1 (type: int) - 1 _col0 (type: int), _col1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col1 <> _col2) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col2 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reducer 6 - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product -PREHOOK: query: select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 -#### A masked pattern was here #### -POSTHOOK: query: select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 -#### A masked pattern was here #### -4 1 -PREHOOK: query: drop table t1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t1 -PREHOOK: Output: default@t1 -POSTHOOK: query: drop table t1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t1 -POSTHOOK: Output: default@t1 -PREHOOK: query: drop table t2 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t2 -PREHOOK: Output: default@t2 -POSTHOOK: query: drop table t2 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t2 -POSTHOOK: Output: default@t2 diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out index 471c2ccd94..28f9a2019e 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out @@ -155,22 +155,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '9') and (value = value)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -440,13 +440,13 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (p_mfgr = p_mfgr) (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + predicate: p_mfgr is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: p_mfgr (type: string), p_size (type: int) sort order: ++ Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 Reducer 2 Reduce Operator Tree: @@ -474,7 +474,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) outputColumnNames: _col2, _col5 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -495,25 +495,25 @@ STAGE PLANS: window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (rank_window_0 <= 2) (type: boolean) - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col5 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(_col1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int) Reducer 5 Reduce Operator Tree: @@ -522,16 +522,16 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string), _col0 (type: int) sort order: ++ Map-reduce partition columns: _col1 (type: string), _col0 (type: int) - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -563,21 +563,21 @@ Manufacturer#2 almond aquamarine midnight light salmon 2 Manufacturer#3 almond antique misty red olive 1 Manufacturer#4 almond aquamarine yellow dodger mint 7 Manufacturer#5 almond antique sky peru orange 2 -PREHOOK: query: explain -select * -from src b +PREHOOK: query: explain +select * +from src b where b.key in - (select distinct a.key - from src a + (select distinct a.key + from src a where b.value = a.value and a.key > '9' ) PREHOOK: type: QUERY -POSTHOOK: query: explain -select * -from src b +POSTHOOK: query: explain +select * +from src b where b.key in - (select distinct a.key - from src a + (select distinct a.key + from src a where b.value = a.value and a.key > '9' ) POSTHOOK: type: QUERY @@ -612,22 +612,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '9') and (value = value)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -652,21 +652,21 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select * -from src b +PREHOOK: query: select * +from src b where b.key in - (select distinct a.key - from src a + (select distinct a.key + from src a where b.value = a.value and a.key > '9' ) PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select * -from src b +POSTHOOK: query: select * +from src b where b.key in - (select distinct a.key - from src a + (select distinct a.key + from src a where b.value = a.value and a.key > '9' ) POSTHOOK: type: QUERY @@ -683,6 +683,143 @@ POSTHOOK: Input: default@src 97 val_97 98 val_98 98 val_98 +PREHOOK: query: explain +select * +from src b +where b.key in + (select distinct a.key + from src a + where b.value <> a.key and a.key > '9' + ) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * +from src b +where b.key in + (select distinct a.key + from src a + where b.value <> a.key and a.key > '9' + ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Map 3 (GROUP, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key > '9') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col1 <> _col3) (type: boolean) + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * +from src b +where b.key in + (select distinct a.key + from src a + where b.value <> a.key and a.key > '9' + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * +from src b +where b.key in + (select distinct a.key + from src a + where b.value <> a.key and a.key > '9' + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +90 val_90 +90 val_90 +90 val_90 +92 val_92 +95 val_95 +95 val_95 +96 val_96 +97 val_97 +97 val_97 +98 val_98 +98 val_98 PREHOOK: query: select p_mfgr, p_name, p_size from part where part.p_size in @@ -1864,18 +2001,18 @@ STAGE PLANS: alias: p Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((p_partkey = p_partkey) and (p_size = p_size)) (type: boolean) - Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: NONE + predicate: (p_partkey is not null and p_size is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: p_partkey (type: int), p_name (type: string), p_size (type: int) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int) sort order: +++ Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int) - Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -1899,16 +2036,16 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col0 (type: int), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int), _col0 (type: string), _col2 (type: int) sort order: +++ Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col2 (type: int) - Statistics: Num rows: 3 Data size: 363 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -3102,22 +3239,22 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((p_mfgr = p_mfgr) and (p_size < 10)) (type: boolean) - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + predicate: ((p_size < 10) and p_mfgr is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: p_mfgr (type: string), p_name (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -4999,3 +5136,373 @@ POSTHOOK: query: drop table tt POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@tt POSTHOOK: Output: default@tt +Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +PREHOOK: query: explain select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 1), Reducer 7 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Reducer 4 (GROUP, 2) + Reducer 7 <- Map 6 (GROUP, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string), _col5 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col5 (type: int) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: int) + Map 6 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string) + outputColumnNames: p_name + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_name (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string), _col5 (type: int) + 1 _col1 (type: string), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 185 Data size: 45180 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 185 Data size: 45180 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 <> _col2) (type: boolean) + Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col2 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(_col1) + keys: _col2 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int) + Reducer 5 + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 169 Data size: 41073 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 169 Data size: 41073 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) + Statistics: Num rows: 169 Data size: 41073 Basic stats: COMPLETE Column stats: NONE + Reducer 7 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +PREHOOK: query: select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu +Warning: Shuffle Join JOIN[16][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +PREHOOK: query: explain select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2) + Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 1), Reducer 8 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 9 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2) + Reducer 6 <- Reducer 5 (GROUP, 2) + Reducer 8 <- Map 7 (GROUP, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: string), _col5 (type: int) + sort order: ++ + Map-reduce partition columns: _col4 (type: string), _col5 (type: int) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: p + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_name (type: string), p_type (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: part + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_type (type: string) + outputColumnNames: p_type + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: p_type (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map 9 + Map Operator Tree: + TableScan + alias: pp + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_type is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int) + Reducer 2 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col4 (type: string), _col5 (type: int) + 1 _col1 (type: string), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 203 Data size: 49563 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 203 Data size: 49563 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col2 <> _col0) (type: boolean) + Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string) + Reducer 5 + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col2, _col4 + Statistics: Num rows: 371 Data size: 90361 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col2 (type: string), _col4 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 371 Data size: 90361 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 371 Data size: 90361 Basic stats: COMPLETE Column stats: NONE + Reducer 6 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 185 Data size: 45058 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 185 Data size: 45058 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) + Statistics: Num rows: 185 Data size: 45058 Basic stats: COMPLETE Column stats: NONE + Reducer 8 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join JOIN[16][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +PREHOOK: query: select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ +110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously +112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +132666 almond aquamarine rose maroon antique Manufacturer#2 Brand#24 SMALL POLISHED NICKEL 25 MED BOX 1698.66 even +144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about +146985 almond aquamarine midnight light salmon Manufacturer#2 Brand#23 MEDIUM BURNISHED COPPER 2 SM CASE 2031.98 s cajole caref +15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu +155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra +17273 almond antique forest lavender goldenrod Manufacturer#3 Brand#35 PROMO ANODIZED TIN 14 JUMBO CASE 1190.27 along the +17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve +191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle +192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir +195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de +33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful +40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s +42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl +45261 almond aquamarine floral ivory bisque Manufacturer#4 Brand#42 SMALL PLATED STEEL 27 WRAP CASE 1206.26 careful +48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i +49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick +65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr +78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith +85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull +86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully +90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl diff --git a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out index 7d93e260f6..a81f20bf2e 100644 --- a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out @@ -354,9 +354,9 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualStringScalar(col 14, val AIR) -> boolean, FilterLongColEqualLongColumn(col 3, col 3) -> boolean) -> boolean - predicate: ((l_linenumber = l_linenumber) and (l_shipmode = 'AIR')) (type: boolean) - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualStringScalar(col 14, val AIR) -> boolean, SelectColumnIsNotNull(col 3) -> boolean) -> boolean + predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean) + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: l_orderkey (type: int), l_linenumber (type: int) outputColumnNames: _col0, _col1 @@ -364,7 +364,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0, 3] - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Group By Operator Group By Vectorization: className: VectorGroupByOperator @@ -377,7 +377,7 @@ STAGE PLANS: keys: _col0 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator Spark Hash Table Sink Vectorization: className: VectorSparkHashTableSinkOperator diff --git a/ql/src/test/results/clientpositive/subquery_exists.q.out b/ql/src/test/results/clientpositive/subquery_exists.q.out index c9f2a79041..f19d8a961a 100644 --- a/ql/src/test/results/clientpositive/subquery_exists.q.out +++ b/ql/src/test/results/clientpositive/subquery_exists.q.out @@ -40,22 +40,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = key) and (value = value) and (value > 'val_9')) (type: boolean) - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + predicate: ((value > 'val_9') and key is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -1023,413 +1023,3 @@ POSTHOOK: query: drop table t POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@t POSTHOOK: Output: default@t -PREHOOK: query: drop table if exists tx1 -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table if exists tx1 -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table tx1 (a integer,b integer) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@tx1 -POSTHOOK: query: create table tx1 (a integer,b integer) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@tx1 -PREHOOK: query: insert into tx1 values (1, 1), - (1, 2), - (1, 3) -PREHOOK: type: QUERY -PREHOOK: Output: default@tx1 -POSTHOOK: query: insert into tx1 values (1, 1), - (1, 2), - (1, 3) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@tx1 -POSTHOOK: Lineage: tx1.a EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: tx1.b EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -PREHOOK: query: select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -PREHOOK: type: QUERY -PREHOOK: Input: default@tx1 -#### A masked pattern was here #### -POSTHOOK: query: select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@tx1 -#### A masked pattern was here #### -3 3 -PREHOOK: query: explain select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -PREHOOK: type: QUERY -POSTHOOK: query: explain select count(*) as result,3 as expected from tx1 u - where exists (select * from tx1 v where u.a=v.a and u.b <> v.b) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-4 is a root stage - Stage-3 depends on stages: Stage-4 - Stage-1 depends on stages: Stage-3 - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - alias: u - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: a, b - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int), b (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: v - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) - TableScan - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col3 <> _col1) (type: boolean) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: int), _col3 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: u - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 _col0 (type: int), _col1 (type: int) - 1 _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), 3 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: drop table tx1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@tx1 -PREHOOK: Output: default@tx1 -POSTHOOK: query: drop table tx1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@tx1 -POSTHOOK: Output: default@tx1 -PREHOOK: query: create table t1(i int, j int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t1 -POSTHOOK: query: create table t1(i int, j int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t1 -PREHOOK: query: insert into t1 values(4,1) -PREHOOK: type: QUERY -PREHOOK: Output: default@t1 -POSTHOOK: query: insert into t1 values(4,1) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@t1 -POSTHOOK: Lineage: t1.i EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: t1.j EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -PREHOOK: query: create table t2(i int, j int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@t2 -POSTHOOK: query: create table t2(i int, j int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@t2 -PREHOOK: query: insert into t2 values(4,2),(4,3),(4,5) -PREHOOK: type: QUERY -PREHOOK: Output: default@t2 -POSTHOOK: query: insert into t2 values(4,2),(4,3),(4,5) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@t2 -POSTHOOK: Lineage: t2.i EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: t2.j EXPRESSION [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product -PREHOOK: query: explain select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-3 is a root stage - Stage-2 depends on stages: Stage-3 - Stage-1 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: j (type: int) - outputColumnNames: j - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - alias: t2 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 3 Data size: 9 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int) - TableScan - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col1 <> _col2) (type: boolean) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), _col2 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: t1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 _col0 (type: int), _col1 (type: int) - 1 _col0 (type: int), _col1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 23 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product -PREHOOK: query: select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -PREHOOK: type: QUERY -PREHOOK: Input: default@t1 -PREHOOK: Input: default@t2 -#### A masked pattern was here #### -POSTHOOK: query: select * from t1 where t1.i in (select t2.i from t2 where t2.j <> t1.j) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@t1 -POSTHOOK: Input: default@t2 -#### A masked pattern was here #### -4 1 -PREHOOK: query: drop table t1 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t1 -PREHOOK: Output: default@t1 -POSTHOOK: query: drop table t1 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t1 -POSTHOOK: Output: default@t1 -PREHOOK: query: drop table t2 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@t2 -PREHOOK: Output: default@t2 -POSTHOOK: query: drop table t2 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@t2 -POSTHOOK: Output: default@t2 diff --git a/ql/src/test/results/clientpositive/subquery_exists_having.q.out b/ql/src/test/results/clientpositive/subquery_exists_having.q.out index 2c41ff6c33..f9d347dc09 100644 --- a/ql/src/test/results/clientpositive/subquery_exists_having.q.out +++ b/ql/src/test/results/clientpositive/subquery_exists_having.q.out @@ -74,22 +74,22 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = key) and (value > 'val_9')) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: ((value > 'val_9') and key is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -192,33 +192,33 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = key) and (value > 'val_9')) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: ((value > 'val_9') and key is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 583 Data size: 6193 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 291 Data size: 3091 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 874 Data size: 9284 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 999 Data size: 10612 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 @@ -235,7 +235,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 874 Data size: 9284 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 999 Data size: 10612 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Semi Join 0 to 1 diff --git a/ql/src/test/results/clientpositive/subquery_in_having.q.out b/ql/src/test/results/clientpositive/subquery_in_having.q.out index 6893442b61..8ccc185949 100644 --- a/ql/src/test/results/clientpositive/subquery_in_having.q.out +++ b/ql/src/test/results/clientpositive/subquery_in_having.q.out @@ -1776,11 +1776,14 @@ POSTHOOK: type: QUERY POSTHOOK: Output: default@src_null POSTHOOK: Lineage: src_null.key SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] POSTHOOK: Lineage: src_null.value SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -Warning: Map Join MAPJOIN[185][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[186][bigTable=?] in task 'Stage-8:MAPRED' is a cross product -Warning: Map Join MAPJOIN[187][bigTable=?] in task 'Stage-9:MAPRED' is a cross product -Warning: Map Join MAPJOIN[188][bigTable=?] in task 'Stage-14:MAPRED' is a cross product -Warning: Map Join MAPJOIN[189][bigTable=?] in task 'Stage-15:MAPRED' is a cross product +Warning: Map Join MAPJOIN[146][bigTable=?] in task 'Stage-18:MAPRED' is a cross product +Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[147][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[148][bigTable=?] in task 'Stage-21:MAPRED' is a cross product +Warning: Shuffle Join JOIN[32][tables = [$hdt$_2, $hdt$_3]] in Stage 'Stage-8:MAPRED' is a cross product +Warning: Map Join MAPJOIN[149][bigTable=?] in task 'Stage-12:MAPRED' is a cross product +Warning: Map Join MAPJOIN[150][bigTable=?] in task 'Stage-24:MAPRED' is a cross product +Warning: Shuffle Join JOIN[72][tables = [$hdt$_3, $hdt$_4]] in Stage 'Stage-13:MAPRED' is a cross product PREHOOK: query: explain select key, value, count(*) from src_null b @@ -1796,39 +1799,44 @@ group by key, value having count(*) not in (select count(*) from src_null s1 where s1.key > '9' and s1.value <> b.value group by s1.key ) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-6 is a root stage - Stage-27 depends on stages: Stage-6 - Stage-1 depends on stages: Stage-27 - Stage-2 depends on stages: Stage-1 - Stage-20 depends on stages: Stage-2, Stage-8 , consists of Stage-26, Stage-3 + Stage-5 is a root stage + Stage-19 depends on stages: Stage-5 , consists of Stage-27, Stage-1 + Stage-27 has a backup stage: Stage-1 + Stage-18 depends on stages: Stage-27 + Stage-2 depends on stages: Stage-1, Stage-18 + Stage-17 depends on stages: Stage-2, Stage-7 , consists of Stage-26, Stage-3 Stage-26 has a backup stage: Stage-3 - Stage-19 depends on stages: Stage-26 - Stage-4 depends on stages: Stage-3, Stage-14, Stage-19 + Stage-16 depends on stages: Stage-26 + Stage-4 depends on stages: Stage-3, Stage-12, Stage-16 Stage-3 - Stage-12 is a root stage - Stage-29 depends on stages: Stage-12 - Stage-9 depends on stages: Stage-29 - Stage-10 depends on stages: Stage-9 - Stage-28 depends on stages: Stage-10 - Stage-8 depends on stages: Stage-28 - Stage-18 is a root stage - Stage-31 depends on stages: Stage-18 - Stage-15 depends on stages: Stage-31 - Stage-16 depends on stages: Stage-15 - Stage-30 depends on stages: Stage-16 - Stage-14 depends on stages: Stage-30 + Stage-1 + Stage-10 is a root stage + Stage-22 depends on stages: Stage-10 , consists of Stage-29, Stage-8 + Stage-29 has a backup stage: Stage-8 + Stage-21 depends on stages: Stage-29 + Stage-9 depends on stages: Stage-8, Stage-21 + Stage-28 depends on stages: Stage-9 + Stage-7 depends on stages: Stage-28 + Stage-8 + Stage-15 is a root stage + Stage-25 depends on stages: Stage-15 , consists of Stage-31, Stage-13 + Stage-31 has a backup stage: Stage-13 + Stage-24 depends on stages: Stage-31 + Stage-14 depends on stages: Stage-13, Stage-24 + Stage-30 depends on stages: Stage-14 + Stage-12 depends on stages: Stage-30 + Stage-13 Stage-0 depends on stages: Stage-4 STAGE PLANS: - Stage: Stage-6 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan - alias: b + alias: src_null Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: value + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: value (type: string) @@ -1846,34 +1854,35 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-19 + Conditional Operator Stage: Stage-27 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_0:$hdt$_1:$hdt$_1:src_null + $hdt$_0:$INTNAME Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_0:$hdt$_1:$hdt$_1:src_null + $hdt$_0:$INTNAME TableScan - alias: src_null - Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 - 1 + HashTable Sink Operator + keys: + 0 + 1 - Stage: Stage-1 + Stage: Stage-18 Map Reduce Map Operator Tree: TableScan @@ -1883,110 +1892,36 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - TableScan - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col0 <> _col1) (type: boolean) + Map Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + residual filter predicates: {(_col1 <> _col2)} Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col1 + Filter Operator + predicate: _col3 is null (type: boolean) Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: string) - mode: hash - outputColumnNames: _col0 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Group By Operator + aggregations: count() + keys: _col1 (type: string), _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Reduce Operator Tree: - Demux Operator - Statistics: Num rows: 2 Data size: 17461 Basic stats: COMPLETE Column stats: NONE - Mux Operator - Statistics: Num rows: 3 Data size: 26191 Basic stats: COMPLETE Column stats: NONE - Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Filter Operator - predicate: _col3 is null (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col1 (type: string), _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8730 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8730 Basic stats: COMPLETE Column stats: NONE - Mux Operator - Statistics: Num rows: 3 Data size: 26191 Basic stats: COMPLETE Column stats: NONE - Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Filter Operator - predicate: _col3 is null (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col1 (type: string), _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -1996,7 +1931,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -2004,11 +1939,11 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2016,7 +1951,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-20 + Stage: Stage-17 Conditional Operator Stage: Stage-26 @@ -2033,7 +1968,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) - Stage: Stage-19 + Stage: Stage-16 Map Reduce Map Operator Tree: TableScan @@ -2044,7 +1979,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col4, _col5 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 12805 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2062,19 +1997,19 @@ STAGE PLANS: key expressions: _col1 (type: string), _col2 (type: bigint) sort order: ++ Map-reduce partition columns: _col1 (type: string), _col2 (type: bigint) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 12805 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col4 (type: bigint), _col5 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Demux Operator - Statistics: Num rows: 2 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 30267 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 3 Data size: 8731 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 45400 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Outer Join 0 to 1 @@ -2101,20 +2036,20 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: bigint) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2910 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 15133 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: bigint), _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2910 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 15133 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2910 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 15133 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), true (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 2910 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 15133 Basic stats: COMPLETE Column stats: NONE Mux Operator - Statistics: Num rows: 3 Data size: 8731 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 45400 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Left Outer Join 0 to 1 @@ -2146,14 +2081,14 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col2 (type: bigint) TableScan Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reduce Operator Tree: Join Operator @@ -2163,7 +2098,7 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col4, _col5 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 12805 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2171,15 +2106,63 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-12 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan alias: b Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string) - outputColumnNames: value + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: boolean) + Reduce Operator Tree: + Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + residual filter predicates: {(_col1 <> _col2)} + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col3 is null (type: boolean) + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col1 (type: string), _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-10 + Map Reduce + Map Operator Tree: + TableScan + alias: src_null + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: value (type: string) @@ -2197,34 +2180,35 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-22 + Conditional Operator Stage: Stage-29 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_1:$hdt$_1:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:src_null + $hdt$_1:$hdt$_1:$hdt$_2:$hdt$_2:$INTNAME Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_1:$hdt$_1:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:src_null + $hdt$_1:$hdt$_1:$hdt$_2:$hdt$_2:$INTNAME TableScan - alias: src_null - Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 - 1 + HashTable Sink Operator + keys: + 0 + 1 - Stage: Stage-9 + Stage: Stage-21 Map Reduce Map Operator Tree: TableScan @@ -2234,110 +2218,37 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - TableScan - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col0 <> _col1) (type: boolean) + Map Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + residual filter predicates: {(_col1 <> _col2)} Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col1 + Filter Operator + predicate: _col3 is null (type: boolean) Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: string) - mode: hash - outputColumnNames: _col0 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Group By Operator + keys: _col1 (type: string), _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Reduce Operator Tree: - Demux Operator - Statistics: Num rows: 2 Data size: 17461 Basic stats: COMPLETE Column stats: NONE - Mux Operator - Statistics: Num rows: 3 Data size: 26191 Basic stats: COMPLETE Column stats: NONE - Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Filter Operator - predicate: _col3 is null (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - keys: _col1 (type: string), _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8730 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8730 Basic stats: COMPLETE Column stats: NONE - Mux Operator - Statistics: Num rows: 3 Data size: 26191 Basic stats: COMPLETE Column stats: NONE - Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Filter Operator - predicate: _col3 is null (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - keys: _col1 (type: string), _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-10 + Stage: Stage-9 Map Reduce Map Operator Tree: TableScan @@ -2345,22 +2256,22 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col1 (type: string) mode: complete outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2391,7 +2302,7 @@ STAGE PLANS: 0 1 - Stage: Stage-8 + Stage: Stage-7 Map Reduce Map Operator Tree: TableScan @@ -2402,25 +2313,25 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col1 <> _col2) (type: boolean) - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: _col2 (type: string), _col0 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Local Work: Map Reduce Local Work @@ -2430,17 +2341,17 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col1, _col2 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), count(_col2) keys: _col1 (type: string) mode: complete outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2448,15 +2359,62 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-18 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan alias: b Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string) - outputColumnNames: value + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: boolean) + Reduce Operator Tree: + Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + residual filter predicates: {(_col1 <> _col2)} + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col3 is null (type: boolean) + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col1 (type: string), _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-15 + Map Reduce + Map Operator Tree: + TableScan + alias: src_null + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: value (type: string) @@ -2474,34 +2432,35 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-25 + Conditional Operator Stage: Stage-31 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_2:$hdt$_2:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:$hdt$_4:$hdt$_4:src_null + $hdt$_2:$hdt$_2:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:$INTNAME Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_2:$hdt$_2:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:$hdt$_4:$hdt$_4:src_null + $hdt$_2:$hdt$_2:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:$INTNAME TableScan - alias: src_null - Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 - 1 + HashTable Sink Operator + keys: + 0 + 1 - Stage: Stage-15 + Stage: Stage-24 Map Reduce Map Operator Tree: TableScan @@ -2511,110 +2470,37 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - TableScan - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col0 <> _col1) (type: boolean) + Map Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + residual filter predicates: {(_col1 <> _col2)} Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string) - outputColumnNames: _col1 + Filter Operator + predicate: _col3 is null (type: boolean) Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: string) - mode: hash - outputColumnNames: _col0 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Group By Operator + keys: _col1 (type: string), _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work - Reduce Operator Tree: - Demux Operator - Statistics: Num rows: 2 Data size: 17461 Basic stats: COMPLETE Column stats: NONE - Mux Operator - Statistics: Num rows: 3 Data size: 26191 Basic stats: COMPLETE Column stats: NONE - Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Filter Operator - predicate: _col3 is null (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - keys: _col1 (type: string), _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8730 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8730 Basic stats: COMPLETE Column stats: NONE - Mux Operator - Statistics: Num rows: 3 Data size: 26191 Basic stats: COMPLETE Column stats: NONE - Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Filter Operator - predicate: _col3 is null (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Group By Operator - keys: _col1 (type: string), _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-16 + Stage: Stage-14 Map Reduce Map Operator Tree: TableScan @@ -2622,22 +2508,22 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col1 (type: string) mode: complete outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2668,7 +2554,7 @@ STAGE PLANS: 0 1 - Stage: Stage-14 + Stage: Stage-12 Map Reduce Map Operator Tree: TableScan @@ -2679,25 +2565,25 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col1 <> _col2) (type: boolean) - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col2 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() keys: _col2 (type: string), _col0 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Local Work: Map Reduce Local Work @@ -2707,16 +2593,16 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col1, _col2 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col1 (type: string), _col2 (type: bigint) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 5821 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 17462 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -2724,17 +2610,68 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-13 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 5820 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: boolean) + Reduce Operator Tree: + Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + residual filter predicates: {(_col1 <> _col2)} + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col3 is null (type: boolean) + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col1 (type: string), _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 11641 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: ListSink -Warning: Map Join MAPJOIN[185][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[186][bigTable=?] in task 'Stage-8:MAPRED' is a cross product -Warning: Map Join MAPJOIN[187][bigTable=?] in task 'Stage-9:MAPRED' is a cross product -Warning: Map Join MAPJOIN[188][bigTable=?] in task 'Stage-14:MAPRED' is a cross product -Warning: Map Join MAPJOIN[189][bigTable=?] in task 'Stage-15:MAPRED' is a cross product +Warning: Map Join MAPJOIN[146][bigTable=?] in task 'Stage-18:MAPRED' is a cross product +Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[147][bigTable=?] in task 'Stage-7:MAPRED' is a cross product +Warning: Map Join MAPJOIN[148][bigTable=?] in task 'Stage-21:MAPRED' is a cross product +Warning: Shuffle Join JOIN[32][tables = [$hdt$_2, $hdt$_3]] in Stage 'Stage-8:MAPRED' is a cross product +Warning: Map Join MAPJOIN[149][bigTable=?] in task 'Stage-12:MAPRED' is a cross product +Warning: Map Join MAPJOIN[150][bigTable=?] in task 'Stage-24:MAPRED' is a cross product +Warning: Shuffle Join JOIN[72][tables = [$hdt$_3, $hdt$_4]] in Stage 'Stage-13:MAPRED' is a cross product PREHOOK: query: select key, value, count(*) from src_null b where NOT EXISTS (select key from src_null where src_null.value <> b.value) diff --git a/ql/src/test/results/clientpositive/subquery_notexists.q.out b/ql/src/test/results/clientpositive/subquery_notexists.q.out index 329573e8e1..a6175f8fec 100644 --- a/ql/src/test/results/clientpositive/subquery_notexists.q.out +++ b/ql/src/test/results/clientpositive/subquery_notexists.q.out @@ -29,28 +29,28 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = key) and (value = value) and (value > 'val_2')) (type: boolean) - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + predicate: ((value > 'val_2') and key is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -78,7 +78,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) Reduce Operator Tree: Join Operator @@ -280,37 +280,37 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((value = value) and (value > 'val_2')) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (value > 'val_2') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: value (type: string), key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col1 (type: string) mode: complete outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -339,7 +339,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Operator Tree: Join Operator diff --git a/ql/src/test/results/clientpositive/subquery_notexists_having.q.out b/ql/src/test/results/clientpositive/subquery_notexists_having.q.out index 4d2b2fc873..0d90e1b548 100644 --- a/ql/src/test/results/clientpositive/subquery_notexists_having.q.out +++ b/ql/src/test/results/clientpositive/subquery_notexists_having.q.out @@ -72,7 +72,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) Reduce Operator Tree: Join Operator @@ -105,28 +105,28 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key = key) and (value = value) and (value > 'val_12')) (type: boolean) - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + predicate: ((value > 'val_12') and key is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -255,7 +255,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: boolean) Reduce Operator Tree: Join Operator @@ -288,37 +288,37 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((value = value) and (value > 'val_12')) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: (value > 'val_12') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: value (type: string), key (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col1 (type: string) mode: complete outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), true (type: boolean) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git a/ql/src/test/results/clientpositive/subquery_notin_having.q.out b/ql/src/test/results/clientpositive/subquery_notin_having.q.out index c321fe69ed..433609d016 100644 --- a/ql/src/test/results/clientpositive/subquery_notin_having.q.out +++ b/ql/src/test/results/clientpositive/subquery_notin_having.q.out @@ -286,7 +286,7 @@ STAGE PLANS: key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reduce Operator Tree: Join Operator @@ -319,7 +319,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: double) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: double) - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) Reduce Operator Tree: Join Operator @@ -375,18 +375,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col2 - _col1) > 600.0) and (_col1 = _col1)) (type: boolean) - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col2 - _col1) > 600.0) and _col1 is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), count(_col0) keys: _col1 (type: double) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -402,7 +402,7 @@ STAGE PLANS: key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -410,7 +410,7 @@ STAGE PLANS: keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -448,19 +448,19 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((_col2 - _col1) > 600.0) and (_col1 = _col1)) (type: boolean) - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + predicate: (((_col2 - _col1) > 600.0) and _col1 is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: double), true (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out index 5c306f6b47..cc2188cbb5 100644 --- a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out +++ b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out @@ -62,22 +62,22 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '9') and (value = value)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -86,10 +86,10 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: string) 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -129,22 +129,22 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '9') and (value = value)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -210,22 +210,22 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '9') and (value = value)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -284,20 +284,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (p_mfgr = p_mfgr) (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + predicate: p_mfgr is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: p_mfgr (type: string), p_size (type: int) sort order: ++ Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: p_name (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) outputColumnNames: _col1, _col2, _col5 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -318,20 +318,20 @@ STAGE PLANS: window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (rank_window_0 <= 2) (type: boolean) - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), count(_col0) keys: _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -347,7 +347,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reduce Operator Tree: Group By Operator @@ -355,7 +355,7 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -384,7 +384,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reduce Operator Tree: Join Operator @@ -417,7 +417,7 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: boolean) Reduce Operator Tree: Join Operator @@ -450,20 +450,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (p_mfgr = p_mfgr) (type: boolean) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + predicate: p_mfgr is not null (type: boolean) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: p_mfgr (type: string), p_size (type: int) sort order: ++ Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: p_name (type: string) Reduce Operator Tree: Select Operator expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) outputColumnNames: _col1, _col2, _col5 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -484,19 +484,19 @@ STAGE PLANS: window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (rank_window_0 <= 2) (type: boolean) - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -512,20 +512,20 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: diff --git a/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out index 4d252eb5ff..faf624b4e0 100644 --- a/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out +++ b/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out @@ -361,12 +361,12 @@ STAGE PLANS: alias: lineitem Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((l_linenumber = l_linenumber) and (l_shipmode = 'AIR')) (type: boolean) - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean) + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: l_orderkey (type: int), l_linenumber (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE Group By Operator Group By Vectorization: groupByMode: HASH @@ -377,7 +377,7 @@ STAGE PLANS: keys: _col0 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col1 (type: int), _col4 (type: int)