diff --git a/pom.xml b/pom.xml index 351df449d6..480369b06a 100644 --- a/pom.xml +++ b/pom.xml @@ -127,7 +127,7 @@ 1.12.0 1.8.2 0.8.0.RELEASE - 1.19.0 + 1.21.0 4.2.4 4.1.17 4.1.19 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java index f50779d8ef..7b09737cec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java @@ -113,9 +113,8 @@ public RelBuilder filter(Iterable predicates) { @Override public RelBuilder empty() { final RelNode input = build(); - final RelNode sort = HiveRelFactories.HIVE_SORT_FACTORY.createSort( - input, RelCollations.of(), null, literal(0)); - return this.push(sort); + final RelNode filter = HiveRelFactories.HIVE_FILTER_FACTORY.createFilter(input, literal(false)); + return this.push(filter); } public static SqlFunction getFloorSqlFunction(TimeUnitRange flag) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java index d96b1dc022..e361b19e1a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java @@ -123,8 +123,9 @@ public RelNode createProject(RelNode child, * . */ private static class HiveFilterFactoryImpl implements FilterFactory { + // XXX: SJC variablesSet is ignored? @Override - public RelNode createFilter(RelNode child, RexNode condition) { + public RelNode createFilter(RelNode child, RexNode condition, Set variablesSet) { RelOptCluster cluster = child.getCluster(); HiveFilter filter = new HiveFilter(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, condition); return filter; @@ -153,6 +154,11 @@ public RelNode createFilter(RelNode child, RexNode condition) { @Override public RelNode createJoin(RelNode left, RelNode right, RexNode condition, JoinRelType joinType, Set variablesStopped, boolean semiJoinDone) { + if (joinType == JoinRelType.SEMI) { + final JoinInfo joinInfo = JoinInfo.of(left, right, condition); + final RelOptCluster cluster = left.getCluster(); + return HiveSemiJoin.getSemiJoin(cluster, left.getTraitSet(), left, right, condition); + } return HiveJoin.getJoin(left.getCluster(), left, right, condition, joinType); } @@ -161,6 +167,11 @@ public RelNode createJoin(RelNode left, RelNode right, RexNode condition, Set variablesSet, JoinRelType joinType, boolean semiJoinDone) { // According to calcite, it is going to be removed before Calcite-2.0 // TODO: to handle CorrelationId + if (joinType == JoinRelType.SEMI) { + final JoinInfo joinInfo = JoinInfo.of(left, right, condition); + final RelOptCluster cluster = left.getCluster(); + return HiveSemiJoin.getSemiJoin(cluster, left.getTraitSet(), left, right, condition); + } return HiveJoin.getJoin(left.getCluster(), left, right, condition, joinType); } } @@ -176,8 +187,7 @@ public RelNode createSemiJoin(RelNode left, RelNode right, RexNode condition) { final JoinInfo joinInfo = JoinInfo.of(left, right, condition); final RelOptCluster cluster = left.getCluster(); - return HiveSemiJoin.getSemiJoin(cluster, left.getTraitSet(), left, right, condition, - joinInfo.leftKeys, joinInfo.rightKeys); + return HiveSemiJoin.getSemiJoin(cluster, left.getTraitSet(), left, right, condition); } } @@ -197,15 +207,11 @@ public RelNode createSort(RelNode input, RelCollation collation, RexNode offset, private static class HiveAggregateFactoryImpl implements AggregateFactory { @Override - public RelNode createAggregate(RelNode child, boolean indicator, + public RelNode createAggregate(RelNode child, ImmutableBitSet groupSet, ImmutableList groupSets, List aggCalls) { - if (indicator) { - throw new IllegalStateException("Hive does not support indicator columns but Calcite " - + "created an Aggregate operator containing them"); - } - return new HiveAggregate(child.getCluster(), child.getTraitSet(), child, - groupSet, groupSets, aggCalls); + return new HiveAggregate(child.getCluster(), child.getTraitSet(), child, + groupSet, groupSets, aggCalls); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptMaterializationValidator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptMaterializationValidator.java index 110136ddcd..25ee38fbd4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptMaterializationValidator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptMaterializationValidator.java @@ -108,7 +108,7 @@ public RelNode visit(HiveFilter filter) { @Override public RelNode visit(HiveJoin join) { - if (join.getJoinType() != JoinRelType.INNER) { + if (join.getJoinType() != JoinRelType.INNER && !join.isSemiJoin()) { setAutomaticRewritingInvalidReason(join.getJoinType() + " join type is not supported by rewriting algorithm."); } checkExpr(join.getCondition()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java index b8380d63cd..fdbcf4304d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java @@ -547,7 +547,7 @@ public static PKFKJoinInfo extractPKFKJoin( final PKFKJoinInfo cannotExtract = PKFKJoinInfo.of(false, null, null); - if (joinType != JoinRelType.INNER) { + if (joinType != JoinRelType.INNER && !join.isSemiJoin()) { // If it is not an inner, we transform it as the metadata // providers for expressions do not pull information through // outer join (as it would not be correct) @@ -741,7 +741,7 @@ public static RewritablePKFKJoinInfo isRewritablePKFKJoin(Join join, final RelNode nonFkInput = leftInputPotentialFK ? join.getRight() : join.getLeft(); final RewritablePKFKJoinInfo nonRewritable = RewritablePKFKJoinInfo.of(false, null); - if (joinType != JoinRelType.INNER) { + if (joinType != JoinRelType.INNER && !join.isSemiJoin()) { // If it is not an inner, we transform it as the metadata // providers for expressions do not pull information through // outer join (as it would not be correct) @@ -848,7 +848,7 @@ public static RewritablePKFKJoinInfo isRewritablePKFKJoin(Join join, if (ecT.getEquivalenceClassesMap().containsKey(uniqueKeyColumnRef) && ecT.getEquivalenceClassesMap().get(uniqueKeyColumnRef).contains(foreignKeyColumnRef)) { if (foreignKeyColumnType.isNullable()) { - if (joinType == JoinRelType.INNER) { + if (joinType == JoinRelType.INNER || join.isSemiJoin()) { // If it is nullable and it is an INNER, we just need a IS NOT NULL filter RexNode originalCondOp = refToRex.get(foreignKeyColumnRef); assert originalCondOp != null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java index 05d1dc6cf2..f4988e7d3a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveSubQRemoveRelBuilder.java @@ -50,7 +50,6 @@ import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.server.CalciteServerStatement; -import org.apache.calcite.sql.SemiJoinType; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; @@ -1141,11 +1140,10 @@ public HiveSubQRemoveRelBuilder join(JoinRelType joinType, RexNode condition, } if(createSemiJoin) { join = correlateFactory.createCorrelate(left.rel, right.rel, id, - requiredColumns, SemiJoinType.SEMI); + requiredColumns, JoinRelType.SEMI); } else { join = correlateFactory.createCorrelate(left.rel, right.rel, id, - requiredColumns, SemiJoinType.of(joinType)); - + requiredColumns, joinType); } } else { join = joinFactory.createJoin(left.rel, right.rel, condition, @@ -1297,9 +1295,8 @@ private boolean allNull(Object[] values, int column, int columnCount) { */ public HiveSubQRemoveRelBuilder empty() { final RelNode input = build(); - final RelNode sort = HiveRelFactories.HIVE_SORT_FACTORY.createSort( - input, RelCollations.of(), null, literal(0)); - return this.push(sort); + final RelNode filter = HiveRelFactories.HIVE_FILTER_FACTORY.createFilter(input, literal(false)); + return this.push(filter); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java index 50466e001b..6b841a5a29 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java @@ -53,14 +53,9 @@ public HiveAggregate(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, @Override public Aggregate copy(RelTraitSet traitSet, RelNode input, - boolean indicator, ImmutableBitSet groupSet, + ImmutableBitSet groupSet, List groupSets, List aggCalls) { - if (indicator) { - throw new IllegalStateException("Hive does not support indicator columns but tried " - + "to create an Aggregate operator containing them"); - } - return new HiveAggregate(getCluster(), traitSet, input, - groupSet, groupSets, aggCalls); + return new HiveAggregate(getCluster(), traitSet, input, groupSet, groupSets, aggCalls); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java index 7a8cf0aa32..705644700c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java @@ -225,7 +225,7 @@ public JoinPredicateInfo getJoinPredicateInfo() { private boolean containsOuter() { for (JoinRelType joinType : joinTypes) { - if (joinType != JoinRelType.INNER) { + if (joinType != JoinRelType.INNER && joinType != JoinRelType.SEMI) { return true; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java index d70ead428d..1bdfea3e3d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java @@ -24,9 +24,9 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.InvalidRelException; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinInfo; import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.ImmutableIntList; @@ -35,8 +35,9 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRulesRegistry; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; -public class HiveSemiJoin extends SemiJoin implements HiveRelNode { +public class HiveSemiJoin extends Join implements HiveRelNode { private final RexNode joinFilter; @@ -46,12 +47,9 @@ public static HiveSemiJoin getSemiJoin( RelTraitSet traitSet, RelNode left, RelNode right, - RexNode condition, - ImmutableIntList leftKeys, - ImmutableIntList rightKeys) { + RexNode condition) { try { - HiveSemiJoin semiJoin = new HiveSemiJoin(cluster, traitSet, left, right, - condition, leftKeys, rightKeys); + HiveSemiJoin semiJoin = new HiveSemiJoin(cluster, traitSet, left, right, condition); return semiJoin; } catch (InvalidRelException | CalciteSemanticException e) { throw new RuntimeException(e); @@ -62,10 +60,8 @@ protected HiveSemiJoin(RelOptCluster cluster, RelTraitSet traitSet, RelNode left, RelNode right, - RexNode condition, - ImmutableIntList leftKeys, - ImmutableIntList rightKeys) throws InvalidRelException, CalciteSemanticException { - super(cluster, traitSet, left, right, condition, leftKeys, rightKeys); + RexNode condition) throws InvalidRelException, CalciteSemanticException { + super(cluster, traitSet, left, right, condition, JoinRelType.SEMI, Sets.newHashSet()); final List systemFieldList = ImmutableList.of(); List> joinKeyExprs = new ArrayList>(); List filterNulls = new ArrayList(); @@ -81,12 +77,11 @@ public RexNode getJoinFilter() { } @Override - public SemiJoin copy(RelTraitSet traitSet, RexNode condition, + public HiveSemiJoin copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { try { final JoinInfo joinInfo = JoinInfo.of(left, right, condition); - HiveSemiJoin semijoin = new HiveSemiJoin(getCluster(), traitSet, left, right, condition, - joinInfo.leftKeys, joinInfo.rightKeys); + HiveSemiJoin semijoin = new HiveSemiJoin(getCluster(), traitSet, left, right, condition); // If available, copy state to registry for optimization rules HiveRulesRegistry registry = semijoin.getCluster().getPlanner().getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java index ed6659c6cc..b7f4ae8504 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java @@ -100,7 +100,7 @@ public void onMatch(RelOptRuleCall call) { // If it is not an inner join, we do not push the // aggregate operator - if (join.getJoinType() != JoinRelType.INNER) { + if (join.getJoinType() != JoinRelType.INNER && !join.isSemiJoin()) { return; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java index 4e66de3812..b2ff255d7c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinAddNotNullRule.java @@ -91,8 +91,8 @@ public void onMatch(RelOptRuleCall call) { Set leftPushedPredicates = Sets.newHashSet(registry.getPushedPredicates(join, 0)); Set rightPushedPredicates = Sets.newHashSet(registry.getPushedPredicates(join, 1)); - boolean genPredOnLeft = join.getJoinType() == JoinRelType.RIGHT || join.getJoinType() == JoinRelType.INNER; - boolean genPredOnRight = join.getJoinType() == JoinRelType.LEFT || join.getJoinType() == JoinRelType.INNER; + boolean genPredOnLeft = join.getJoinType() == JoinRelType.RIGHT || join.getJoinType() == JoinRelType.INNER || join.isSemiJoin(); + boolean genPredOnRight = join.getJoinType() == JoinRelType.LEFT || join.getJoinType() == JoinRelType.INNER || join.isSemiJoin(); RexNode newLeftPredicate = getNewPredicate(join, registry, joinPredInfo, leftPushedPredicates, genPredOnLeft, 0); RexNode newRightPredicate = getNewPredicate(join, registry, joinPredInfo, rightPushedPredicates, genPredOnRight, 1); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java index c735df81eb..a657d1313f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java @@ -117,7 +117,7 @@ public void onMatch(RelOptRuleCall call) { // These boolean values represent corresponding left, right input which is potential FK boolean leftInputPotentialFK = topRefs.intersects(leftBits); boolean rightInputPotentialFK = topRefs.intersects(rightBits); - if (leftInputPotentialFK && rightInputPotentialFK && joinType == JoinRelType.INNER) { + if (leftInputPotentialFK && rightInputPotentialFK && (joinType == JoinRelType.INNER || joinType == JoinRelType.SEMI)) { // Both inputs are referenced. Before making a decision, try to swap // references in join condition if it is an inner join, i.e. if a join // condition column is referenced above the join, then we can just @@ -198,6 +198,7 @@ public void onMatch(RelOptRuleCall call) { final Mode mode; switch (joinType) { + case SEMI: case INNER: if (leftInputPotentialFK && rightInputPotentialFK) { // Bails out as it references columns from both sides (or no columns) @@ -283,4 +284,4 @@ public void onMatch(RelOptRuleCall call) { // Transforms LEFT/RIGHT outer join into INNER join TRANSFORM } -} \ No newline at end of file +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectJoinTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectJoinTransposeRule.java index 38759c0525..545255cf7c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectJoinTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectJoinTransposeRule.java @@ -24,7 +24,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.rules.PushProjector; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexNode; @@ -77,7 +76,7 @@ public void onMatch(RelOptRuleCall call) { Project origProj = call.rel(0); final Join join = call.rel(1); - if (join instanceof SemiJoin) { + if (join.isSemiJoin()) { return; // TODO: support SemiJoin } // locate all fields referenced in the projection and join condition; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java index 86b79140c6..068b6875b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java @@ -76,7 +76,6 @@ import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexVisitorImpl; -import org.apache.calcite.sql.SemiJoinType; import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; @@ -503,9 +502,6 @@ public Frame decorrelateRel(Values rel) { * @param rel Aggregate to rewrite */ public Frame decorrelateRel(Aggregate rel) throws SemanticException{ - if (rel.getGroupType() != Aggregate.Group.SIMPLE) { - throw new AssertionError(Bug.CALCITE_461_FIXED); - } // // Rewrite logic: // @@ -688,9 +684,6 @@ private static RexLiteral projectedLiteral(RelNode rel, int i) { } public Frame decorrelateRel(HiveAggregate rel) throws SemanticException{ - if (rel.getGroupType() != Aggregate.Group.SIMPLE) { - throw new AssertionError(Bug.CALCITE_461_FIXED); - } // // Rewrite logic: // @@ -1248,7 +1241,7 @@ public Frame decorrelateRel(HiveFilter rel) throws SemanticException { } if(oldInput instanceof LogicalCorrelate - && ((LogicalCorrelate) oldInput).getJoinType() == SemiJoinType.SEMI + && ((LogicalCorrelate) oldInput).getJoinType() == JoinRelType.SEMI && !cm.mapRefRelToCorRef.containsKey(rel)) { // this conditions need to be pushed into semi-join since this condition // corresponds to IN @@ -1261,7 +1254,7 @@ public Frame decorrelateRel(HiveFilter rel) throws SemanticException { RexUtil.composeConjunction(rexBuilder, conditions, false); RelNode newRel = HiveSemiJoin.getSemiJoin(frame.r.getCluster(), frame.r.getTraitSet(), - join.getLeft(), join.getRight(), condition, join.getLeftKeys(), join.getRightKeys()); + join.getLeft(), join.getRight(), condition); return register(rel, newRel, frame.oldToNewOutputs, frame.corDefOutputs); } @@ -1320,7 +1313,7 @@ public Frame decorrelateRel(Filter rel) { } if(oldInput instanceof LogicalCorrelate - && ((LogicalCorrelate) oldInput).getJoinType() == SemiJoinType.SEMI + && ((LogicalCorrelate) oldInput).getJoinType() == JoinRelType.SEMI && !cm.mapRefRelToCorRef.containsKey(rel)) { // this conditions need to be pushed into semi-join since this condition // corresponds to IN @@ -1332,7 +1325,7 @@ public Frame decorrelateRel(Filter rel) { final RexNode condition = RexUtil.composeConjunction(rexBuilder, conditions, false); RelNode newRel = HiveSemiJoin.getSemiJoin(frame.r.getCluster(), frame.r.getTraitSet(), - join.getLeft(), join.getRight(), condition, join.getLeftKeys(), join.getRightKeys()); + join.getLeft(), join.getRight(), condition); return register(rel, newRel, frame.oldToNewOutputs, frame.corDefOutputs); } @@ -1455,14 +1448,13 @@ public Frame decorrelateRel(LogicalCorrelate rel) { RelNode newJoin = null; // this indicates original query was either correlated EXISTS or IN - if(rel.getJoinType() == SemiJoinType.SEMI) { + if(rel.getJoinType() == JoinRelType.SEMI) { final List leftKeys = new ArrayList(); final List rightKeys = new ArrayList(); RelNode[] inputRels = new RelNode[] {leftFrame.r, rightFrame.r}; newJoin = HiveSemiJoin.getSemiJoin(rel.getCluster(), - rel.getCluster().traitSetOf(HiveRelNode.CONVENTION), leftFrame.r, rightFrame.r, - condition, ImmutableIntList.copyOf(leftKeys), ImmutableIntList.copyOf(rightKeys)); + rel.getCluster().traitSetOf(HiveRelNode.CONVENTION), leftFrame.r, rightFrame.r, condition); } else { // Right input positions are shifted by newLeftFieldCount. @@ -1473,7 +1465,7 @@ public Frame decorrelateRel(LogicalCorrelate rel) { } newJoin = relBuilder.push(leftFrame.r).push(rightFrame.r) - .join(rel.getJoinType().toJoinType(), condition).build(); + .join(rel.getJoinType(), condition).build(); } valueGen.pop(); @@ -1720,7 +1712,7 @@ private RelNode aggregateCorrelatorOutput( Project project, Set isCount) { final RelNode left = correlate.getLeft(); - final JoinRelType joinType = correlate.getJoinType().toJoinType(); + final JoinRelType joinType = correlate.getJoinType(); // now create the new project final List> newProjects = Lists.newArrayList(); @@ -2258,10 +2250,10 @@ public void onMatch(RelOptRuleCall call) { // Aggregate (groupby (0) single_value()) // Project-A (may reference coVar) // RightInputRel - if(correlate.getJoinType() != SemiJoinType.LEFT) { + if(correlate.getJoinType() != JoinRelType.LEFT) { return; } - final JoinRelType joinType = correlate.getJoinType().toJoinType(); + final JoinRelType joinType = correlate.getJoinType(); // corRel.getCondition was here, however Correlate was updated so it // never includes a join condition. The code was not modified for brevity. @@ -2470,11 +2462,11 @@ public void onMatch(RelOptRuleCall call) { return; } - if(correlate.getJoinType() != SemiJoinType.LEFT) { + if(correlate.getJoinType() != JoinRelType.LEFT) { return; } - final JoinRelType joinType = correlate.getJoinType().toJoinType(); + final JoinRelType joinType = correlate.getJoinType(); // corRel.getCondition was here, however Correlate was updated so it // never includes a join condition. The code was not modified for brevity. RexNode joinCond = rexBuilder.makeLiteral(true); @@ -2877,11 +2869,11 @@ private void onMatch2( return; } - if(correlate.getJoinType() != SemiJoinType.LEFT) { + if(correlate.getJoinType() != JoinRelType.LEFT) { return; } - JoinRelType joinType = correlate.getJoinType().toJoinType(); + JoinRelType joinType = correlate.getJoinType(); // corRel.getCondition was here, however Correlate was updated so it // never includes a join condition. The code was not modified for brevity. RexNode joinCond = rexBuilder.makeLiteral(true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveGBYSemiJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveGBYSemiJoinRule.java index 4992e702f9..82704a2ebd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveGBYSemiJoinRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRemoveGBYSemiJoinRule.java @@ -51,9 +51,8 @@ public HiveRemoveGBYSemiJoinRule() { @Override public void onMatch(RelOptRuleCall call) { final HiveSemiJoin semijoin= call.rel(0); - if(semijoin.getJoinType() != JoinRelType.INNER) { - return; - } + assert semijoin.getJoinType() == JoinRelType.SEMI; + final RelNode left = call.rel(1); final Aggregate rightAggregate= call.rel(2); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSemiJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSemiJoinRule.java index e63f163b24..7842b12b52 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSemiJoinRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSemiJoinRule.java @@ -118,7 +118,7 @@ protected void perform(final RelOptRuleCall call, final ImmutableBitSet topRefs, call.transformTo(topOperator.copy(topOperator.getTraitSet(), ImmutableList.of(left))); return; } - if (join.getJoinType() != JoinRelType.INNER) { + if (join.getJoinType() != JoinRelType.INNER && !join.isSemiJoin()) { return; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/jdbc/HiveJdbcImplementor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/jdbc/HiveJdbcImplementor.java index db955b9a9d..eefd2bd2c3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/jdbc/HiveJdbcImplementor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/jdbc/HiveJdbcImplementor.java @@ -109,7 +109,7 @@ public HiveJdbcImplementor(SqlDialect dialect, JavaTypeFactory typeFactory) { SqlNode sqlCondition = null; SqlLiteral condType = JoinConditionType.ON.symbol(POS); JoinType joinType = joinType(e.getJoinType()); - if (e.getJoinType() == JoinRelType.INNER && e.getCondition().isAlwaysTrue()) { + if ((e.getJoinType() == JoinRelType.INNER || e.isSemiJoin()) && e.getCondition().isAlwaysTrue()) { joinType = JoinType.COMMA; condType = JoinConditionType.NONE.symbol(POS); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java index b2b2f3c2d8..12353dd344 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java @@ -25,7 +25,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdDistinctRowCount; @@ -41,6 +40,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveCost; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.plan.ColStatistics; @@ -85,16 +85,13 @@ public static Double getDistinctRowCount(RelNode r, RelMetadataQuery mq, int ind @Override public Double getDistinctRowCount(Join rel, RelMetadataQuery mq, ImmutableBitSet groupKey, RexNode predicate) { + if (rel instanceof HiveSemiJoin) { + return super.getDistinctRowCount(rel, mq, groupKey, predicate); + } + if (rel instanceof HiveJoin) { - HiveJoin hjRel = (HiveJoin) rel; - //TODO: Improve this - if (rel instanceof SemiJoin) { - return mq.getDistinctRowCount(hjRel.getLeft(), groupKey, - rel.getCluster().getRexBuilder().makeLiteral(true)); - } else { - return getJoinDistinctRowCount(mq, rel, rel.getJoinType(), - groupKey, predicate, true); - } + return getJoinDistinctRowCount(mq, rel, rel.getJoinType(), + groupKey, predicate, true); } return mq.getDistinctRowCount(rel, groupKey, predicate); @@ -116,7 +113,7 @@ public Double getDistinctRowCount(Join rel, RelMetadataQuery mq, ImmutableBitSet * otherwise use left NDV * right NDV. * @return number of distinct rows */ - public static Double getJoinDistinctRowCount(RelMetadataQuery mq, + private static Double getJoinDistinctRowCount(RelMetadataQuery mq, RelNode joinRel, JoinRelType joinType, ImmutableBitSet groupKey, RexNode predicate, boolean useMaxNdv) { Double distRowCount = null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java index a137bdf94f..4b01c776e5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java @@ -39,7 +39,6 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.metadata.BuiltInMetadata; import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; @@ -316,7 +315,7 @@ public RelOptPredicateList getPredicates(Union union, RelMetadataQuery mq) { public JoinConditionBasedPredicateInference(Join joinRel, RexNode lPreds, RexNode rPreds) { - this(joinRel, joinRel instanceof SemiJoin, lPreds, rPreds); + this(joinRel, (joinRel instanceof Join) && ((Join) joinRel).isSemiJoin(), lPreds, rPreds); } private JoinConditionBasedPredicateInference(Join joinRel, boolean isSemiJoin, @@ -416,6 +415,7 @@ public RelOptPredicateList inferPredicates( switch (joinType) { case INNER: case LEFT: + case SEMI: infer(leftPreds, allExprsDigests, inferredPredicates, nonFieldsPredicates, includeEqualityInference, joinType == JoinRelType.LEFT ? rightFieldsBitSet @@ -425,6 +425,7 @@ public RelOptPredicateList inferPredicates( switch (joinType) { case INNER: case RIGHT: + case SEMI: infer(rightPreds, allExprsDigests, inferredPredicates, nonFieldsPredicates, includeEqualityInference, joinType == JoinRelType.RIGHT ? leftFieldsBitSet @@ -453,7 +454,7 @@ public RelOptPredicateList inferPredicates( } } - if (joinType == JoinRelType.INNER && !nonFieldsPredicates.isEmpty()) { + if ((joinType == JoinRelType.INNER || joinType == JoinRelType.SEMI) && !nonFieldsPredicates.isEmpty()) { // Predicates without field references can be pushed to both inputs final Set leftPredsSet = new HashSet( Lists.transform(leftPreds, HiveCalciteUtil.REX_STR_FN)); @@ -471,15 +472,13 @@ public RelOptPredicateList inferPredicates( switch (joinType) { case INNER: - Iterable pulledUpPredicates; - if (isSemiJoin) { - pulledUpPredicates = Iterables.concat(leftPreds, leftInferredPredicates); - } else { - pulledUpPredicates = Iterables.concat(leftPreds, rightPreds, + Iterable pulledUpPredicates = Iterables.concat(leftPreds, rightPreds, RelOptUtil.conjunctions(joinRel.getCondition()), inferredPredicates); - } return RelOptPredicateList.of(rexBuilder, pulledUpPredicates, leftInferredPredicates, rightInferredPredicates); + case SEMI: + return RelOptPredicateList.of(rexBuilder, Iterables.concat(leftPreds, leftInferredPredicates), + leftInferredPredicates, rightInferredPredicates); case LEFT: return RelOptPredicateList.of(rexBuilder, leftPreds, EMPTY_LIST, rightInferredPredicates); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java index d881fbd952..c0601c0685 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java @@ -30,7 +30,6 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; @@ -75,6 +74,9 @@ protected HiveRelMdRowCount() { @Override public Double getRowCount(Join join, RelMetadataQuery mq) { + if (join.isSemiJoin()) { + return getRowCountSemi(join, mq); + } // Try to infer from constraints first final Pair constraintBasedResult = constraintsBasedAnalyzeJoinForPKFK(join, mq); @@ -117,8 +119,7 @@ public Double getRowCount(Join join, RelMetadataQuery mq) { return rowCount; } - @Override - public Double getRowCount(SemiJoin rel, RelMetadataQuery mq) { + public Double getRowCountSemi(Join rel, RelMetadataQuery mq) { PKFKRelationInfo pkfk = analyzeJoinForPKFK(rel, mq); if (pkfk != null) { double selectivity = pkfk.pkInfo.selectivity * pkfk.ndvScalingFactor; @@ -249,7 +250,7 @@ public static PKFKRelationInfo analyzeJoinForPKFK(Join joinRel, RelMetadataQuery // @todo: remove this. 8/28/14 hb // for now adding because RelOptUtil.classifyFilters has an assertion about // column counts that is not true for semiJoins. - if (joinRel instanceof SemiJoin) { + if (joinRel.isSemiJoin()) { return null; } @@ -356,7 +357,7 @@ public static PKFKRelationInfo analyzeJoinForPKFK(Join joinRel, RelMetadataQuery */ public static Pair constraintsBasedAnalyzeJoinForPKFK(Join join, RelMetadataQuery mq) { - if (join instanceof SemiJoin) { + if (join.isSemiJoin()) { // TODO: Support semijoin return null; } @@ -391,9 +392,9 @@ public static PKFKRelationInfo analyzeJoinForPKFK(Join joinRel, RelMetadataQuery return null; } - boolean leftIsKey = (join.getJoinType() == JoinRelType.INNER || join.getJoinType() == JoinRelType.RIGHT) + boolean leftIsKey = (join.getJoinType() == JoinRelType.INNER || join.isSemiJoin() || join.getJoinType() == JoinRelType.RIGHT) && leftInputResult.isPkFkJoin; - boolean rightIsKey = (join.getJoinType() == JoinRelType.INNER || join.getJoinType() == JoinRelType.LEFT) + boolean rightIsKey = (join.getJoinType() == JoinRelType.INNER || join.isSemiJoin() || join.getJoinType() == JoinRelType.LEFT) && rightInputResult.isPkFkJoin; if (!leftIsKey && !rightIsKey) { // Nothing to do here, bail out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java index f6a6cf4f03..1724ab1d21 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java @@ -25,7 +25,6 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdSelectivity; import org.apache.calcite.rel.metadata.RelMdUtil; @@ -65,7 +64,7 @@ public Double getSelectivity(HiveTableScan t, RelMetadataQuery mq, RexNode predi } public Double getSelectivity(Join j, RelMetadataQuery mq, RexNode predicate) { - if (j.getJoinType().equals(JoinRelType.INNER)) { + if (j.getJoinType().equals(JoinRelType.INNER) || j.isSemiJoin()) { return computeInnerJoinSelectivity(j, mq, predicate); } else if (j.getJoinType().equals(JoinRelType.LEFT) || j.getJoinType().equals(JoinRelType.RIGHT)) { @@ -143,7 +142,7 @@ private Double computeInnerJoinSelectivity(Join j, RelMetadataQuery mq, RexNode ndvEstimate = exponentialBackoff(peLst, colStatMap); } - if (j instanceof SemiJoin) { + if (j.isSemiJoin()) { ndvEstimate = Math.min(mq.getRowCount(j.getLeft()), ndvEstimate); } else if (j instanceof HiveJoin) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java index 893cb9975c..fd3553d5f8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java @@ -20,7 +20,6 @@ import java.util.List; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdSize; import org.apache.calcite.rel.metadata.RelMetadataProvider; @@ -31,6 +30,7 @@ import org.apache.calcite.util.ImmutableNullableList; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.plan.ColStatistics; @@ -78,8 +78,7 @@ private HiveRelMdSize() {} return list.build(); } - @Override - public List averageColumnSizes(SemiJoin rel, RelMetadataQuery mq) { + public List averageColumnSizes(HiveSemiJoin rel, RelMetadataQuery mq) { final RelNode left = rel.getLeft(); final List lefts = mq.getAverageColumnSizes(left); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java index c4c771e490..7328b72bba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java @@ -155,17 +155,15 @@ public static ASTNode table(final RelNode scan) { return b.node(); } - public static ASTNode join(ASTNode left, ASTNode right, JoinRelType joinType, ASTNode cond, - boolean semiJoin) { + public static ASTNode join(ASTNode left, ASTNode right, JoinRelType joinType, ASTNode cond) { ASTBuilder b = null; switch (joinType) { + case SEMI: + b = ASTBuilder.construct(HiveParser.TOK_LEFTSEMIJOIN, "TOK_LEFTSEMIJOIN"); + break; case INNER: - if (semiJoin) { - b = ASTBuilder.construct(HiveParser.TOK_LEFTSEMIJOIN, "TOK_LEFTSEMIJOIN"); - } else { - b = ASTBuilder.construct(HiveParser.TOK_JOIN, "TOK_JOIN"); - } + b = ASTBuilder.construct(HiveParser.TOK_JOIN, "TOK_JOIN"); break; case LEFT: b = ASTBuilder.construct(HiveParser.TOK_LEFTOUTERJOIN, "TOK_LEFTOUTERJOIN"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java index 6c4edeb905..718a2d0bd0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java @@ -36,7 +36,6 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.TableFunctionScan; import org.apache.calcite.rel.core.TableScan; @@ -378,7 +377,7 @@ private QueryBlockInfo convertSource(RelNode r) throws CalciteSemanticException QueryBlockInfo right = convertSource(join.getRight()); s = new Schema(left.schema, right.schema); ASTNode cond = join.getCondition().accept(new RexVisitor(s, false, r.getCluster().getRexBuilder())); - boolean semiJoin = join instanceof SemiJoin; + boolean semiJoin = join.isSemiJoin(); if (join.getRight() instanceof Join && !semiJoin) { // should not be done for semijoin since it will change the semantics // Invert join inputs; this is done because otherwise the SemanticAnalyzer @@ -391,9 +390,9 @@ private QueryBlockInfo convertSource(RelNode r) throws CalciteSemanticException } else { type = join.getJoinType(); } - ast = ASTBuilder.join(right.ast, left.ast, type, cond, semiJoin); + ast = ASTBuilder.join(right.ast, left.ast, type, cond); } else { - ast = ASTBuilder.join(left.ast, right.ast, join.getJoinType(), cond, semiJoin); + ast = ASTBuilder.join(left.ast, right.ast, join.getJoinType(), cond); } if (semiJoin) { s = left.schema; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java index 165cb9efcf..509a070b84 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java @@ -37,7 +37,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.SemiJoin; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; @@ -175,8 +174,8 @@ OpAttr dispatch(RelNode rn) throws SemanticException { return visit((HiveMultiJoin) rn); } else if (rn instanceof HiveJoin) { return visit((HiveJoin) rn); - } else if (rn instanceof SemiJoin) { - return visit((SemiJoin)rn); + } else if (rn instanceof HiveSemiJoin) { + return visit((HiveSemiJoin) rn); } else if (rn instanceof HiveFilter) { return visit((HiveFilter) rn); } else if (rn instanceof HiveSortLimit) { @@ -332,8 +331,7 @@ OpAttr visit(HiveJoin joinRel) throws SemanticException { return translateJoin(joinRel); } - - OpAttr visit(SemiJoin joinRel) throws SemanticException { + OpAttr visit(HiveSemiJoin joinRel) throws SemanticException { return translateJoin(joinRel); } @@ -365,8 +363,8 @@ private OpAttr translateJoin(RelNode joinRel) throws SemanticException { // 3. Virtual columns Set newVcolsInCalcite = new HashSet(); newVcolsInCalcite.addAll(inputs[0].vcolsInCalcite); - if (joinRel instanceof HiveMultiJoin || - !(joinRel instanceof SemiJoin)) { + if (joinRel instanceof HiveMultiJoin || + !((joinRel instanceof Join) && ((Join) joinRel).isSemiJoin())) { int shift = inputs[0].inputs.get(0).getSchema().getSignature().size(); for (int i = 1; i < inputs.length; i++) { newVcolsInCalcite.addAll(HiveCalciteUtil.shiftVColsSet(inputs[i].vcolsInCalcite, shift)); @@ -904,7 +902,7 @@ private static JoinOperator genJoin(RelNode join, ExprNodeDesc[][] joinExpressio noOuterJoin = !hmj.isOuterJoin(); } else { joinCondns = new JoinCondDesc[1]; - semiJoin = join instanceof SemiJoin; + semiJoin = (join instanceof Join) && ((Join) join).isSemiJoin(); JoinType joinType; if (semiJoin) { joinType = JoinType.LEFTSEMI; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 60cd71583f..9334a8ee19 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -2697,7 +2697,7 @@ private RelNode genJoinRelNode(RelNode leftRel, String leftTableAlias, RelNode r calciteJoinType = JoinRelType.FULL; break; case LEFTSEMI: - calciteJoinType = JoinRelType.INNER; + calciteJoinType = JoinRelType.SEMI; leftSemiJoin = true; break; case INNER: @@ -2731,8 +2731,7 @@ private RelNode genJoinRelNode(RelNode leftRel, String leftTableAlias, RelNode r ImmutableList.of(remainingEquiCond, nonEquiConds), false) : nonEquiConds; topRel = HiveSemiJoin.getSemiJoin(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), - inputRels[0], inputRels[1], calciteJoinCond, ImmutableIntList.copyOf(leftKeys), - ImmutableIntList.copyOf(rightKeys)); + inputRels[0], inputRels[1], calciteJoinCond); // Create join RR: we need to check whether we need to update left RR in case // previous call to projectNonColumnEquiConditions updated it diff --git a/ql/src/test/results/clientpositive/annotate_stats_limit.q.out b/ql/src/test/results/clientpositive/annotate_stats_limit.q.out index ec1dc8bb25..04c436446f 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_limit.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_limit.q.out @@ -152,12 +152,35 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_n5 #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc_n5 + Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/cbo_SortUnionTransposeRule.q.out b/ql/src/test/results/clientpositive/cbo_SortUnionTransposeRule.q.out index 89beaa5d2a..792afd8f78 100644 --- a/ql/src/test/results/clientpositive/cbo_SortUnionTransposeRule.q.out +++ b/ql/src/test/results/clientpositive/cbo_SortUnionTransposeRule.q.out @@ -96,12 +96,55 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@s_n3 #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Union + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + alias: b + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Union + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink @@ -573,12 +616,55 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@s_n3 #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Union + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + alias: b + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + Union + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/concat_op.q.out b/ql/src/test/results/clientpositive/concat_op.q.out index a36379acd6..4442658387 100644 --- a/ql/src/test/results/clientpositive/concat_op.q.out +++ b/ql/src/test/results/clientpositive/concat_op.q.out @@ -311,7 +311,7 @@ POSTHOOK: query: explain formatted select key || value from src POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 9.624,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"key\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"key\",\n \"ndv\": 316\n },\n {\n \"name\": \"value\",\n \"ndv\": 307\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"_o__c0\"\n ],\n \"exprs\": [\n {\n \"op\": \"||\",\n \"operands\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n }\n ],\n \"rowCount\": 500.0\n }\n ]\n}","optimizedSQL":"SELECT `key` || `value` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(key, value) (type: string)","columnExprMap:":{"_col0":"concat(key, value)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 9.624,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"key\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"key\",\n \"ndv\": 316\n },\n {\n \"name\": \"value\",\n \"ndv\": 307\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"_o__c0\"\n ],\n \"exprs\": [\n {\n \"op\": {\n \"name\": \"||\",\n \"kind\": \"OTHER_FUNCTION\",\n \"syntax\": \"SPECIAL\"\n },\n \"operands\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n }\n ],\n \"rowCount\": 500.0\n }\n ]\n}","optimizedSQL":"SELECT `key` || `value` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(key, value) (type: string)","columnExprMap:":{"_col0":"concat(key, value)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} PREHOOK: query: explain formatted select key || value || key from src PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -320,7 +320,7 @@ POSTHOOK: query: explain formatted select key || value || key from src POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 9.624,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"key\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"key\",\n \"ndv\": 316\n },\n {\n \"name\": \"value\",\n \"ndv\": 307\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"_o__c0\"\n ],\n \"exprs\": [\n {\n \"op\": \"||\",\n \"operands\": [\n {\n \"op\": \"||\",\n \"operands\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n }\n ],\n \"rowCount\": 500.0\n }\n ]\n}","optimizedSQL":"SELECT `key` || `value` || `key` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(concat(key, value), key) (type: string)","columnExprMap:":{"_col0":"concat(concat(key, value), key)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 9.624,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"key\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"key\",\n \"ndv\": 316\n },\n {\n \"name\": \"value\",\n \"ndv\": 307\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"_o__c0\"\n ],\n \"exprs\": [\n {\n \"op\": {\n \"name\": \"||\",\n \"kind\": \"OTHER_FUNCTION\",\n \"syntax\": \"SPECIAL\"\n },\n \"operands\": [\n {\n \"op\": {\n \"name\": \"||\",\n \"kind\": \"OTHER_FUNCTION\",\n \"syntax\": \"SPECIAL\"\n },\n \"operands\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n }\n ],\n \"rowCount\": 500.0\n }\n ]\n}","optimizedSQL":"SELECT `key` || `value` || `key` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(concat(key, value), key) (type: string)","columnExprMap:":{"_col0":"concat(concat(key, value), key)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} PREHOOK: query: explain formatted select key || value || key || value from src PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -329,4 +329,4 @@ POSTHOOK: query: explain formatted select key || value || key || value from src POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 9.624,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"key\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"key\",\n \"ndv\": 316\n },\n {\n \"name\": \"value\",\n \"ndv\": 307\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"_o__c0\"\n ],\n \"exprs\": [\n {\n \"op\": \"||\",\n \"operands\": [\n {\n \"op\": \"||\",\n \"operands\": [\n {\n \"op\": \"||\",\n \"operands\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n }\n ],\n \"rowCount\": 500.0\n }\n ]\n}","optimizedSQL":"SELECT `key` || `value` || `key` || `value` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(concat(concat(key, value), key), value) (type: string)","columnExprMap:":{"_col0":"concat(concat(concat(key, value), key), value)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 9.624,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"key\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"key\",\n \"ndv\": 316\n },\n {\n \"name\": \"value\",\n \"ndv\": 307\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"_o__c0\"\n ],\n \"exprs\": [\n {\n \"op\": {\n \"name\": \"||\",\n \"kind\": \"OTHER_FUNCTION\",\n \"syntax\": \"SPECIAL\"\n },\n \"operands\": [\n {\n \"op\": {\n \"name\": \"||\",\n \"kind\": \"OTHER_FUNCTION\",\n \"syntax\": \"SPECIAL\"\n },\n \"operands\": [\n {\n \"op\": {\n \"name\": \"||\",\n \"kind\": \"OTHER_FUNCTION\",\n \"syntax\": \"SPECIAL\"\n },\n \"operands\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n }\n ],\n \"rowCount\": 500.0\n }\n ]\n}","optimizedSQL":"SELECT `key` || `value` || `key` || `value` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(concat(concat(key, value), key), value) (type: string)","columnExprMap:":{"_col0":"concat(concat(concat(key, value), key), value)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} diff --git a/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out b/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out index e05bc13bad..cda99dd3f0 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_expressions.q.out @@ -1348,7 +1348,7 @@ STAGE PLANS: properties: druid.fieldNames vc druid.fieldTypes timestamp - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} druid.query.type scan Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -1532,7 +1532,7 @@ STAGE PLANS: properties: druid.fieldNames vc druid.fieldTypes string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format((div(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),1000) * '1000'),'yyyy-MM-dd HH:mm:ss','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} + druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format((div(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),1000) * '1000'),'yyyy-MM-dd HH:mm:ss','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} druid.query.type groupBy Select Operator expressions: vc (type: string) @@ -1574,7 +1574,7 @@ STAGE PLANS: properties: druid.fieldNames vc druid.fieldTypes string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1Y','','UTC'),'yyyy-MM-dd','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} + druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1Y','','UTC'),'yyyy-MM-dd','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} druid.query.type groupBy Select Operator expressions: vc (type: string) @@ -1629,7 +1629,7 @@ STAGE PLANS: properties: druid.fieldNames vc druid.fieldTypes string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1M','','UTC'),'yyyy-MM-dd','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} + druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1M','','UTC'),'yyyy-MM-dd','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} druid.query.type groupBy Select Operator expressions: vc (type: string) @@ -1684,7 +1684,7 @@ STAGE PLANS: properties: druid.fieldNames vc druid.fieldTypes string - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P3M','','UTC'),'yyyy-MM-dd','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} + druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"STRING"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P3M','','UTC'),'yyyy-MM-dd','UTC')","outputType":"STRING"}],"limitSpec":{"type":"default"},"aggregations":[],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} druid.query.type groupBy Select Operator expressions: vc (type: string) @@ -1773,7 +1773,7 @@ STAGE PLANS: properties: druid.fieldNames vc,$f1 druid.fieldTypes date,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(timestamp_parse(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1M','','UTC'),'yyyy-MM-dd','UTC'),'','UTC'),'P1D','','UTC')","outputType":"LONG"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f1","expression":"(\"cdouble\" * \"cdouble\")"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} + druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(timestamp_parse(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1M','','UTC'),'yyyy-MM-dd','UTC'),'','UTC'),'P1D','','UTC')","outputType":"LONG"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleSum","name":"$f1","expression":"(\"cdouble\" * \"cdouble\")"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} druid.query.type groupBy Select Operator expressions: $f1 (type: double), vc (type: date) @@ -1822,7 +1822,7 @@ STAGE PLANS: properties: druid.fieldNames vc,vc0,vc1,vc2 druid.fieldTypes date,date,date,int - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_shift(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC'),'P1D',CAST((\"cdouble\" / 1000), 'LONG'),'UTC')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"timestamp_shift(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC'),'P1D',-( CAST((\"cdouble\" / 1000), 'LONG') ),'UTC')","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC')","outputType":"LONG"},{"type":"expression","name":"vc2","expression":"CAST((\"cdouble\" / 1000), 'LONG')","outputType":"LONG"}],"columns":["vc","vc0","vc1","vc2"],"resultFormat":"compactedList"} + druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_shift(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'P1D',CAST((\"cdouble\" / 1000), 'LONG'),'UTC')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"timestamp_shift(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'P1D',-( CAST((\"cdouble\" / 1000), 'LONG') ),'UTC')","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC')","outputType":"LONG"},{"type":"expression","name":"vc2","expression":"CAST((\"cdouble\" / 1000), 'LONG')","outputType":"LONG"}],"columns":["vc","vc0","vc1","vc2"],"resultFormat":"compactedList"} druid.query.type scan Statistics: Num rows: 9173 Data size: 1499152 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -2367,7 +2367,7 @@ STAGE PLANS: properties: druid.fieldNames vc druid.fieldTypes timestamp - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} druid.query.type scan Statistics: Num rows: 9173 Data size: 348640 Basic stats: COMPLETE Column stats: NONE Select Operator diff --git a/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out b/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out index 475a40e152..7b80b05b1f 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out @@ -507,7 +507,7 @@ STAGE PLANS: properties: druid.fieldNames vc,vc0,vc1 druid.fieldTypes int,bigint,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"expression","expression":"(CAST(substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2), 'DOUBLE') == 31)"},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'DAY','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"div(timestamp_extract(\"__time\",'DAY','US/Pacific'),7)","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} + druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"expression","expression":"(CAST(substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2), 'DOUBLE') == 31)"},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'DAY','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"div(timestamp_extract(\"__time\",'DAY','US/Pacific'),7)","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} druid.query.type scan Select Operator expressions: vc (type: int), vc0 (type: bigint), vc1 (type: string) @@ -589,7 +589,7 @@ STAGE PLANS: properties: druid.fieldNames vc,vc0,vc1 druid.fieldTypes double,int,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"expression","expression":"(((CAST(timestamp_extract(\"__time\",'MONTH','US/Pacific'), 'DOUBLE') / 4) + 1) == 4)"},{"type":"bound","dimension":"__time","lower":"11","lowerStrict":false,"upper":"12","upperStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"M","timeZone":"US/Pacific","locale":"en-US"}}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"4","outputType":"DOUBLE"},{"type":"expression","name":"vc0","expression":"timestamp_extract(\"__time\",'MONTH','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 5, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} + druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"expression","expression":"(((CAST(timestamp_extract(\"__time\",'MONTH','US/Pacific'), 'DOUBLE') / 4) + 1) == 4)"},{"type":"bound","dimension":"__time","lower":"11","lowerStrict":false,"upper":"12","upperStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"M","timeZone":"US/Pacific","locale":"en-US"}}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"4","outputType":"DOUBLE"},{"type":"expression","name":"vc0","expression":"timestamp_extract(\"__time\",'MONTH','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 5, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} druid.query.type scan Select Operator expressions: vc (type: double), vc0 (type: int), vc1 (type: string) @@ -671,7 +671,7 @@ STAGE PLANS: properties: druid.fieldNames vc,vc0 druid.fieldTypes int,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"bound","dimension":"__time","lower":"1969-01-01T08:00:00.000Z","lowerStrict":false,"ordering":"lexicographic","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"expression","expression":"(CAST(timestamp_extract(\"__time\",'YEAR','US/Pacific'), 'STRING') == '1969')"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'YEAR','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 0, 4)","outputType":"STRING"}],"columns":["vc","vc0"],"resultFormat":"compactedList","limit":1} + druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"bound","dimension":"__time","lower":"1969-01-01T08:00:00.000Z","lowerStrict":false,"ordering":"lexicographic","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"expression","expression":"(CAST(timestamp_extract(\"__time\",'YEAR','US/Pacific'), 'STRING') == '1969')"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'YEAR','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 0, 4)","outputType":"STRING"}],"columns":["vc","vc0"],"resultFormat":"compactedList","limit":1} druid.query.type scan Select Operator expressions: vc (type: int), vc0 (type: string) @@ -748,7 +748,7 @@ STAGE PLANS: properties: druid.fieldNames vc,$f1 druid.fieldTypes date,double - druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC')","outputType":"LONG"}],"limitSpec":{"type":"default","limit":5,"columns":[{"dimension":"vc","direction":"ascending","dimensionOrder":"lexicographic"}]},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cfloat"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} + druid.query.json {"queryType":"groupBy","dataSource":"default.druid_table_alltypesorc","granularity":"all","dimensions":[{"type":"default","dimension":"vc","outputName":"vc","outputType":"LONG"}],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC')","outputType":"LONG"}],"limitSpec":{"type":"default","limit":5,"columns":[{"dimension":"vc","direction":"ascending","dimensionOrder":"lexicographic"}]},"aggregations":[{"type":"doubleSum","name":"$f1","fieldName":"cfloat"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} druid.query.type groupBy Select Operator expressions: vc (type: date), $f1 (type: double) diff --git a/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out b/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out index aced1367d2..9502b522a9 100644 --- a/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out +++ b/ql/src/test/results/clientpositive/druid/druidmini_floorTime.q.out @@ -508,7 +508,7 @@ STAGE PLANS: properties: druid.fieldNames vc,vc0,vc1 druid.fieldTypes int,bigint,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"expression","expression":"(CAST(substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2), 'DOUBLE') == 31)"},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'DAY','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"div(timestamp_extract(\"__time\",'DAY','US/Pacific'),7)","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} + druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"expression","expression":"(CAST(substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2), 'DOUBLE') == 31)"},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'DAY','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"div(timestamp_extract(\"__time\",'DAY','US/Pacific'),7)","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 8, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} druid.query.type scan Select Operator expressions: vc (type: int), vc0 (type: bigint), vc1 (type: string) @@ -590,7 +590,7 @@ STAGE PLANS: properties: druid.fieldNames vc,vc0,vc1 druid.fieldTypes double,int,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"expression","expression":"(((CAST(timestamp_extract(\"__time\",'MONTH','US/Pacific'), 'DOUBLE') / 4) + 1) == 4)"},{"type":"bound","dimension":"__time","lower":"11","lowerStrict":false,"upper":"12","upperStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"M","timeZone":"US/Pacific","locale":"en-US"}}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"4","outputType":"DOUBLE"},{"type":"expression","name":"vc0","expression":"timestamp_extract(\"__time\",'MONTH','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 5, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} + druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"expression","expression":"(((CAST(timestamp_extract(\"__time\",'MONTH','US/Pacific'), 'DOUBLE') / 4) + 1) == 4)"},{"type":"bound","dimension":"__time","lower":"11","lowerStrict":false,"upper":"12","upperStrict":false,"ordering":"numeric","extractionFn":{"type":"timeFormat","format":"M","timeZone":"US/Pacific","locale":"en-US"}}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"4","outputType":"DOUBLE"},{"type":"expression","name":"vc0","expression":"timestamp_extract(\"__time\",'MONTH','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 5, 2)","outputType":"STRING"}],"columns":["vc","vc0","vc1"],"resultFormat":"compactedList","limit":1} druid.query.type scan Select Operator expressions: vc (type: double), vc0 (type: int), vc1 (type: string) @@ -672,7 +672,7 @@ STAGE PLANS: properties: druid.fieldNames vc,vc0 druid.fieldTypes int,string - druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"bound","dimension":"__time","lower":"1969-01-01T08:00:00.000Z","lowerStrict":false,"ordering":"lexicographic","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"expression","expression":"(CAST(timestamp_extract(\"__time\",'YEAR','US/Pacific'), 'STRING') == '1969')"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'YEAR','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 0, 4)","outputType":"STRING"}],"columns":["vc","vc0"],"resultFormat":"compactedList","limit":1} + druid.query.json {"queryType":"scan","dataSource":"default.druid_table_alltypesorc","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"and","fields":[{"type":"bound","dimension":"__time","lower":"1969-01-01T08:00:00.000Z","lowerStrict":false,"ordering":"lexicographic","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"expression","expression":"(CAST(timestamp_extract(\"__time\",'YEAR','US/Pacific'), 'STRING') == '1969')"}]},"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'YEAR','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"substring(timestamp_format(timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','UTC'),'P1D','','UTC'),'yyyy-MM-dd','UTC'), 0, 4)","outputType":"STRING"}],"columns":["vc","vc0"],"resultFormat":"compactedList","limit":1} druid.query.type scan Select Operator expressions: vc (type: int), vc0 (type: string) diff --git a/ql/src/test/results/clientpositive/inputwherefalse.q.out b/ql/src/test/results/clientpositive/inputwherefalse.q.out index a461100565..bee8fd6e42 100644 --- a/ql/src/test/results/clientpositive/inputwherefalse.q.out +++ b/ql/src/test/results/clientpositive/inputwherefalse.q.out @@ -73,12 +73,73 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + filterExpr: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + TableScan + alias: b + filterExpr: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) = 100.0D) (type: boolean) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 22750 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/limit0.q.out b/ql/src/test/results/clientpositive/limit0.q.out index 89a682bdd4..08ee6cb84a 100644 --- a/ql/src/test/results/clientpositive/limit0.q.out +++ b/ql/src/test/results/clientpositive/limit0.q.out @@ -7,12 +7,35 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink @@ -25,12 +48,41 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + filterExpr: (key = '238') (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '238') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: '238' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink @@ -43,12 +95,43 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + filterExpr: (key = '238') (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '238') (type: boolean) + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: '238' (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink @@ -61,12 +144,55 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(value) + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink @@ -79,12 +205,51 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 3 + Statistics: Num rows: 3 Data size: 261 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out index 1c233d01eb..ff4a9e3c8a 100644 --- a/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out +++ b/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out @@ -2256,7 +2256,7 @@ FROM "STORE_SALES" WHERE "ss_customer_sk" IS NOT NULL AND "ss_sold_date_sk" IS NOT NULL) AS "t0" INNER JOIN (SELECT "d_date_sk" FROM "DATE_DIM" -WHERE "d_year" = 1999 AND "d_moy" BETWEEN 1 AND 3 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."ss_sold_date_sk" = "t2"."d_date_sk" +WHERE "d_moy" BETWEEN 1 AND 3 AND "d_year" = 1999 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."ss_sold_date_sk" = "t2"."d_date_sk" hive.sql.query.fieldNames ss_customer_sk0 hive.sql.query.fieldTypes int hive.sql.query.split false @@ -2289,7 +2289,7 @@ FROM "WEB_SALES" WHERE "ws_bill_customer_sk" IS NOT NULL AND "ws_sold_date_sk" IS NOT NULL) AS "t0" INNER JOIN (SELECT "d_date_sk" FROM "DATE_DIM" -WHERE "d_year" = 1999 AND "d_moy" BETWEEN 1 AND 3 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."ws_sold_date_sk" = "t2"."d_date_sk" +WHERE "d_moy" BETWEEN 1 AND 3 AND "d_year" = 1999 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."ws_sold_date_sk" = "t2"."d_date_sk" GROUP BY "t0"."ws_bill_customer_sk" hive.sql.query.fieldNames literalTrue,ws_bill_customer_sk0 hive.sql.query.fieldTypes boolean,int @@ -2318,7 +2318,7 @@ FROM "CATALOG_SALES" WHERE "cs_ship_customer_sk" IS NOT NULL AND "cs_sold_date_sk" IS NOT NULL) AS "t0" INNER JOIN (SELECT "d_date_sk" FROM "DATE_DIM" -WHERE "d_year" = 1999 AND "d_moy" BETWEEN 1 AND 3 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."cs_sold_date_sk" = "t2"."d_date_sk" +WHERE "d_moy" BETWEEN 1 AND 3 AND "d_year" = 1999 AND "d_date_sk" IS NOT NULL) AS "t2" ON "t0"."cs_sold_date_sk" = "t2"."d_date_sk" GROUP BY "t0"."cs_ship_customer_sk" hive.sql.query.fieldNames literalTrue,cs_ship_customer_sk0 hive.sql.query.fieldTypes boolean,int diff --git a/ql/src/test/results/clientpositive/llap/insert_only_empty_query.q.out b/ql/src/test/results/clientpositive/llap/insert_only_empty_query.q.out index bfa72c59a3..7729186421 100644 --- a/ql/src/test/results/clientpositive/llap/insert_only_empty_query.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_only_empty_query.q.out @@ -54,9 +54,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -64,36 +63,22 @@ STAGE PLANS: TableScan alias: source_table Statistics: Num rows: 1 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: string), age (type: int), gpa (type: decimal(3,2)) - outputColumnNames: _col0, _col1, _col2 + Filter Operator + predicate: false (type: boolean) Statistics: Num rows: 1 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 0 + Select Operator + expressions: name (type: string), age (type: int), gpa (type: decimal(3,2)) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - sort order: + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 1 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: decimal(3,2)) + value expressions: _col0 (type: string), _col2 (type: decimal(3,2)) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: int), VALUE._col2 (type: decimal(3,2)) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 0 - Statistics: Num rows: 1 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col2 (type: decimal(3,2)) - Reducer 3 Execution mode: llap Reduce Operator Tree: Select Operator @@ -126,7 +111,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 1500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct) - Reducer 4 + Reducer 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out b/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out index c07ee83216..8c718d953c 100644 --- a/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out @@ -915,12 +915,115 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src3 + filterExpr: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 814 Data size: 289784 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 814 Data size: 289784 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1287 Data size: 687258 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col4 (type: string), _col5 (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out index 041bb28f3a..91b764906b 100644 --- a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out @@ -816,12 +816,54 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out b/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out index 2d2b6eb173..10bd3b3c33 100644 --- a/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out @@ -899,12 +899,54 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out index 1e0aa93123..2d042c31b8 100644 --- a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out +++ b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out @@ -824,12 +824,54 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/llap/semijoin.q.out b/ql/src/test/results/clientpositive/llap/semijoin.q.out index 99ed8de40b..6a0cd4c6ae 100644 --- a/ql/src/test/results/clientpositive/llap/semijoin.q.out +++ b/ql/src/test/results/clientpositive/llap/semijoin.q.out @@ -3277,7 +3277,7 @@ POSTHOOK: Input: default@part #### A masked pattern was here #### CBO PLAN: HiveProject(p_partkey=[$0]) - HiveSemiJoin(condition=[=($1, $2)], joinType=[inner]) + HiveSemiJoin(condition=[=($1, $2)], joinType=[semi]) HiveProject(p_partkey=[$0], p_name=[$1]) HiveFilter(condition=[IS NOT NULL($1)]) HiveTableScan(table=[[default, part]], table:alias=[pp]) @@ -3339,7 +3339,7 @@ POSTHOOK: Input: default@part #### A masked pattern was here #### CBO PLAN: HiveAggregate(group=[{}], agg#0=[count()]) - HiveSemiJoin(condition=[=($0, $1)], joinType=[inner]) + HiveSemiJoin(condition=[=($0, $1)], joinType=[semi]) HiveProject(p_partkey=[$0]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, part]], table:alias=[pp]) diff --git a/ql/src/test/results/clientpositive/llap/subquery_ANY.q.out b/ql/src/test/results/clientpositive/llap/subquery_ANY.q.out index 2c01b01e52..2d402124fc 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_ANY.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_ANY.q.out @@ -88,7 +88,7 @@ POSTHOOK: Input: default@part #### A masked pattern was here #### CBO PLAN: HiveAggregate(group=[{}], agg#0=[count()]) - HiveSemiJoin(condition=[=($0, $1)], joinType=[inner]) + HiveSemiJoin(condition=[=($0, $1)], joinType=[semi]) HiveProject(p_partkey=[$0]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, part]], table:alias=[part]) diff --git a/ql/src/test/results/clientpositive/llap/subquery_in.q.out b/ql/src/test/results/clientpositive/llap/subquery_in.q.out index 76d6400619..95c1b3bd18 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -2818,7 +2818,7 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Left Semi Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col1 (type: bigint) 1 _col0 (type: bigint) @@ -2851,17 +2851,11 @@ STAGE PLANS: expressions: _col1 (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -3050,7 +3044,7 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Left Semi Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col1 (type: bigint) 1 _col0 (type: bigint) @@ -3083,17 +3077,11 @@ STAGE PLANS: expressions: _col1 (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out index 67540b1467..b4c9656f8f 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -1777,7 +1777,7 @@ HiveProject(p_partkey=[$0], p_name=[$1], p_mfgr=[$2], p_brand=[$3], p_type=[$4], HiveTableScan(table=[[default, part_null]], table:alias=[part_null]) HiveProject(p_type=[$0], c=[$1], ck=[$2]) HiveAggregate(group=[{1}], c=[COUNT()], ck=[COUNT($2)]) - HiveSemiJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[inner]) + HiveSemiJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[semi]) HiveProject(p_brand=[$3], p_type=[$4], p_container=[$6]) HiveFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) HiveTableScan(table=[[default, part]], table:alias=[part]) @@ -1786,7 +1786,7 @@ HiveProject(p_partkey=[$0], p_name=[$1], p_mfgr=[$2], p_brand=[$3], p_type=[$4], HiveTableScan(table=[[default, part]], table:alias=[pp]) HiveProject(p_container=[$1], literalTrue=[true], p_type=[$0]) HiveAggregate(group=[{1, 2}]) - HiveSemiJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[inner]) + HiveSemiJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[semi]) HiveProject(p_brand=[$3], p_type=[$4], p_container=[$6]) HiveFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3), IS NOT NULL($6))]) HiveTableScan(table=[[default, part]], table:alias=[part]) diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 9c2f946a5e..052f33fdf6 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -3577,7 +3577,7 @@ almond aquamarine sandy cyan gainsboro almond aquamarine yellow dodger mint almond azure aquamarine papaya violet almond azure blanched chiffon midnight -Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -3747,7 +3747,7 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Left Semi Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col1 (type: bigint) 1 _col0 (type: bigint) @@ -3810,17 +3810,11 @@ STAGE PLANS: expressions: _col1 (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -3828,7 +3822,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select key, count(*) from src where value NOT IN (select key from src) group by key having count(*) in (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -4046,7 +4040,7 @@ STAGE PLANS: Reduce Operator Tree: Merge Join Operator condition map: - Left Semi Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col1 (type: bigint) 1 _col0 (type: bigint) @@ -4112,17 +4106,11 @@ STAGE PLANS: expressions: _col1 (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out index 888e2fa65c..7904875e34 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out @@ -2438,9 +2438,8 @@ POSTHOOK: Input: default@part 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 8' is a cross product -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select key, count(*) from src where value <> (select max(value) from src) group by key having count(*) > (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -2458,54 +2457,29 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 11 <- Map 10 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (XPROD_EDGE), Reducer 6 (XPROD_EDGE) - Reducer 5 <- Reducer 4 (XPROD_EDGE), Reducer 9 (XPROD_EDGE) - Reducer 6 <- Map 1 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (XPROD_EDGE), Reducer 11 (XPROD_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 6 (XPROD_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (XPROD_EDGE), Reducer 8 (XPROD_EDGE) + Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s1 - filterExpr: (key = '90') (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.5 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: true (type: boolean) - minReductionHashAggr: 0.5 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Map 10 + Map 5 Map Operator Tree: TableScan alias: src @@ -2529,66 +2503,74 @@ STAGE PLANS: Map 7 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) + alias: s1 + filterExpr: (key = '90') (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key = '90') (type: boolean) + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: true (type: boolean) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean) + sort order: + + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs - Reducer 11 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: max(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) Reducer 2 - Execution mode: vectorized, llap + Execution mode: llap Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: boolean) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + residual filter predicates: {(_col1 <> _col2)} + Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - minReductionHashAggr: 0.0 + keys: _col0 (type: string) + minReductionHashAggr: 0.5 mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (sq_count_check(_col0) <= 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: bigint) Reducer 4 Execution mode: llap Reduce Operator Tree: @@ -2598,26 +2580,11 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col1, _col2, _col3 - residual filter predicates: {(_col3 > _col1)} + outputColumnNames: _col0, _col1, _col2 + residual filter predicates: {(_col1 > _col2)} Statistics: Num rows: 83 Data size: 8549 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col2 (type: string), _col3 (type: bigint) + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 7885 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -2628,6 +2595,18 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 6 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Reducer 8 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2651,51 +2630,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - residual filter predicates: {(_col1 <> _col2)} - Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 181000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.5 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 9 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -2703,9 +2637,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[59][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 8' is a cross product -Warning: Shuffle Join MERGEJOIN[60][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select key, count(*) from src where value <> (select max(value) from src) group by key having count(*) > (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -7001,8 +6934,7 @@ POSTHOOK: query: drop table tempty_n0 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@tempty_n0 POSTHOOK: Output: default@tempty_n0 -Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[44][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[25][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select key, count(*) from src group by key having count(*) > (select count(*) from src s1 group by 4) PREHOOK: type: QUERY @@ -7023,122 +6955,73 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (XPROD_EDGE), Reducer 6 (XPROD_EDGE) - Reducer 5 <- Reducer 4 (XPROD_EDGE), Reducer 8 (XPROD_EDGE) - Reducer 6 <- Map 1 (SIMPLE_EDGE) - Reducer 8 <- Map 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (XPROD_EDGE), Reducer 5 (XPROD_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - keys: true (type: boolean) - minReductionHashAggr: 0.99 + keys: key (type: string) + minReductionHashAggr: 0.5 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: boolean) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs - Map 7 + Map 4 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.5 + keys: true (type: boolean) + minReductionHashAggr: 0.99 mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: string) + key expressions: _col0 (type: boolean) sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: boolean) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: boolean) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (sq_count_check(_col0) <= 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 5 + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: bigint) + Reducer 3 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -7147,11 +7030,11 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col1, _col2, _col3 - residual filter predicates: {(_col3 > _col1)} + outputColumnNames: _col0, _col1, _col2 + residual filter predicates: {(_col1 > _col2)} Statistics: Num rows: 83 Data size: 8549 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col2 (type: string), _col3 (type: bigint) + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 7885 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -7161,7 +7044,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -7185,22 +7068,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 8 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -7208,8 +7075,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[47][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 6' is a cross product -Warning: Shuffle Join MERGEJOIN[48][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[27][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select key, count(*) from src group by key having count(*) > (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY @@ -7230,11 +7096,8 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (XPROD_EDGE), Reducer 6 (XPROD_EDGE) + Reducer 3 <- Reducer 2 (XPROD_EDGE), Reducer 4 (XPROD_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (XPROD_EDGE), Reducer 7 (XPROD_EDGE) - Reducer 7 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -7259,22 +7122,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 23750 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.5 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (key = '90') (type: boolean) Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE @@ -7320,11 +7167,11 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col1, _col2, _col3 - residual filter predicates: {(_col3 > _col1)} + outputColumnNames: _col0, _col1, _col2 + residual filter predicates: {(_col1 > _col2)} Statistics: Num rows: 83 Data size: 8549 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col2 (type: string), _col3 (type: bigint) + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 7885 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -7335,57 +7182,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 4 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: boolean) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.0 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 5 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (sq_count_check(_col0) <= 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 7 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out b/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out index 4ed197813e..308b24162f 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out @@ -862,12 +862,104 @@ PLAN VECTORIZATION: enabledConditionsMet: [hive.vectorized.execution.enabled IS true] STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 5] + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: double) + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: 0:tinyint, 5:double + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 12 + includeColumns: [0, 5] + dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: zz + reduceColumnSortOrder: ++ + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + dataColumns: KEY.reducesinkkey0:tinyint, KEY.reducesinkkey1:double + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterConstantBooleanVectorExpression(val 0) + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/parquet_vectorization_limit.q.out b/ql/src/test/results/clientpositive/parquet_vectorization_limit.q.out index 67a99838c8..23e4afb848 100644 --- a/ql/src/test/results/clientpositive/parquet_vectorization_limit.q.out +++ b/ql/src/test/results/clientpositive/parquet_vectorization_limit.q.out @@ -567,12 +567,69 @@ PLAN VECTORIZATION: enabledConditionsMet: [hive.vectorized.execution.enabled IS true] STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesparquet + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 5] + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: tinyint), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/perf/spark/query44.q.out b/ql/src/test/results/clientpositive/perf/spark/query44.q.out index c7c7965d91..74ea8f889c 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query44.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query44.q.out @@ -1,5 +1,5 @@ -Warning: Shuffle Join JOIN[35][tables = [$hdt$_2, $hdt$_3, $hdt$_4]] in Work 'Reducer 3' is a cross product -Warning: Shuffle Join JOIN[79][tables = [$hdt$_3, $hdt$_4, $hdt$_5]] in Work 'Reducer 16' is a cross product +Warning: Shuffle Join JOIN[19][tables = [$hdt$_2, $hdt$_3]] in Work 'Reducer 3' is a cross product +Warning: Shuffle Join JOIN[47][tables = [$hdt$_3, $hdt$_4]] in Work 'Reducer 13' is a cross product PREHOOK: query: explain select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * @@ -82,19 +82,17 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 10 <- Map 18 (GROUP, 100) - Reducer 12 <- Map 11 (GROUP, 100) - Reducer 13 <- Reducer 12 (GROUP, 1) - Reducer 15 <- Map 1 (GROUP, 199) - Reducer 16 <- Reducer 13 (PARTITION-LEVEL SORT, 1), Reducer 15 (PARTITION-LEVEL SORT, 1), Reducer 19 (PARTITION-LEVEL SORT, 1) - Reducer 17 <- Reducer 16 (PARTITION-LEVEL SORT, 1009) - Reducer 19 <- Map 18 (GROUP, 100) + Reducer 10 <- Map 15 (GROUP, 100) + Reducer 12 <- Map 1 (GROUP, 199) + Reducer 13 <- Reducer 12 (PARTITION-LEVEL SORT, 1), Reducer 16 (PARTITION-LEVEL SORT, 1) + Reducer 14 <- Reducer 13 (PARTITION-LEVEL SORT, 1009) + Reducer 16 <- Map 15 (GROUP, 100) Reducer 2 <- Map 1 (GROUP, 199) - Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 1), Reducer 13 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1) Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1009) - Reducer 5 <- Reducer 17 (PARTITION-LEVEL SORT, 1009), Reducer 4 (PARTITION-LEVEL SORT, 1009) - Reducer 6 <- Map 23 (PARTITION-LEVEL SORT, 1009), Reducer 5 (PARTITION-LEVEL SORT, 1009) - Reducer 7 <- Map 24 (PARTITION-LEVEL SORT, 1009), Reducer 6 (PARTITION-LEVEL SORT, 1009) + Reducer 5 <- Reducer 14 (PARTITION-LEVEL SORT, 1009), Reducer 4 (PARTITION-LEVEL SORT, 1009) + Reducer 6 <- Map 17 (PARTITION-LEVEL SORT, 1009), Reducer 5 (PARTITION-LEVEL SORT, 1009) + Reducer 7 <- Map 18 (PARTITION-LEVEL SORT, 1009), Reducer 6 (PARTITION-LEVEL SORT, 1009) Reducer 8 <- Reducer 7 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -125,30 +123,7 @@ STAGE PLANS: Statistics: Num rows: 287997817 Data size: 25407250999 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(17,2)), _col2 (type: bigint) Execution mode: vectorized - Map 11 - Map Operator Tree: - TableScan - alias: store_sales - filterExpr: (ss_hdemo_sk is null and (ss_store_sk = 410)) (type: boolean) - Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (ss_hdemo_sk is null and (ss_store_sk = 410)) (type: boolean) - Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE - Execution mode: vectorized - Map 18 + Map 15 Map Operator Tree: TableScan alias: store_sales @@ -175,7 +150,7 @@ STAGE PLANS: Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(17,2)), _col2 (type: bigint) Execution mode: vectorized - Map 23 + Map 17 Map Operator Tree: TableScan alias: i1 @@ -195,7 +170,7 @@ STAGE PLANS: Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized - Map 24 + Map 18 Map Operator Tree: TableScan alias: i2 @@ -232,50 +207,14 @@ STAGE PLANS: predicate: (_col1 is not null and _col2 is not null) (type: boolean) Statistics: Num rows: 71999454 Data size: 6351812727 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (0.9 * (_col1 / _col2)) (type: decimal(38,22)) + expressions: (_col1 / _col2) (type: decimal(37,22)) outputColumnNames: _col0 Statistics: Num rows: 71999454 Data size: 6351812727 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 71999454 Data size: 6351812727 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: decimal(38,22)) + value expressions: _col0 (type: decimal(37,22)) Reducer 12 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: boolean) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 71999454 Data size: 6351812727 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 71999454 Data size: 6351812727 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 13 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (sq_count_check(_col0) <= 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 15 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -295,35 +234,33 @@ STAGE PLANS: sort order: Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: decimal(37,22)) - Reducer 16 + Reducer 13 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 - Inner Join 0 to 2 keys: 0 1 - 2 outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 10367842752596232 Data size: 1922618777862369774 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10367842752596232 Data size: 1839676035841599918 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col1 > _col2) (type: boolean) - Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE + predicate: (_col1 > (0.9 * _col2)) (type: boolean) + Statistics: Num rows: 3455947584198744 Data size: 613225345280533248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 0 (type: int), _col1 (type: decimal(37,22)) sort order: +- Map-reduce partition columns: 0 (type: int) - Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3455947584198744 Data size: 613225345280533248 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: int) - Reducer 17 + Reducer 14 Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: decimal(37,22)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3455947584198744 Data size: 613225345280533248 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -344,21 +281,21 @@ STAGE PLANS: window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3455947584198744 Data size: 613225345280533248 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((rank_window_0 < 11) and _col0 is not null) (type: boolean) - Statistics: Num rows: 1151982528066248 Data size: 213624308651374400 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1151982528066248 Data size: 204408448426844416 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), rank_window_0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1151982528066248 Data size: 213624308651374400 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1151982528066248 Data size: 204408448426844416 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1151982528066248 Data size: 213624308651374400 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1151982528066248 Data size: 204408448426844416 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) - Reducer 19 + Reducer 16 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -375,13 +312,13 @@ STAGE PLANS: predicate: (_col1 is not null and _col2 is not null) (type: boolean) Statistics: Num rows: 71999454 Data size: 6351812727 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (0.9 * (_col1 / _col2)) (type: decimal(38,22)) + expressions: (_col1 / _col2) (type: decimal(37,22)) outputColumnNames: _col0 Statistics: Num rows: 71999454 Data size: 6351812727 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 71999454 Data size: 6351812727 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: decimal(38,22)) + value expressions: _col0 (type: decimal(37,22)) Reducer 2 Execution mode: vectorized Reduce Operator Tree: @@ -407,21 +344,19 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 - Inner Join 0 to 2 keys: 0 1 - 2 outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 10367842752596232 Data size: 1922618777862369774 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10367842752596232 Data size: 1839676035841599918 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col1 > _col2) (type: boolean) - Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE + predicate: (_col1 > (0.9 * _col2)) (type: boolean) + Statistics: Num rows: 3455947584198744 Data size: 613225345280533248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 0 (type: int), _col1 (type: decimal(37,22)) sort order: ++ Map-reduce partition columns: 0 (type: int) - Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3455947584198744 Data size: 613225345280533248 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: int) Reducer 4 @@ -430,7 +365,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: decimal(37,22)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3455947584198744 Data size: 613225345280533248 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -451,19 +386,19 @@ STAGE PLANS: window function: GenericUDAFRankEvaluator window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) isPivotResult: true - Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3455947584198744 Data size: 613225345280533248 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((rank_window_0 < 11) and _col0 is not null) (type: boolean) - Statistics: Num rows: 1151982528066248 Data size: 213624308651374400 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1151982528066248 Data size: 204408448426844416 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), rank_window_0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1151982528066248 Data size: 213624308651374400 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1151982528066248 Data size: 204408448426844416 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1151982528066248 Data size: 213624308651374400 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1151982528066248 Data size: 204408448426844416 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) Reducer 5 Reduce Operator Tree: @@ -474,12 +409,12 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1267180808338276 Data size: 234986744609712256 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1267180808338276 Data size: 224849298143006048 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1267180808338276 Data size: 234986744609712256 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1267180808338276 Data size: 224849298143006048 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col2 (type: int) Reducer 6 Reduce Operator Tree: @@ -490,12 +425,12 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col2, _col5 - Statistics: Num rows: 1393898919384048 Data size: 258485424673204064 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1393898919384048 Data size: 247334233318131680 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) sort order: + Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1393898919384048 Data size: 258485424673204064 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1393898919384048 Data size: 247334233318131680 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: int), _col5 (type: string) Reducer 7 Reduce Operator Tree: @@ -506,15 +441,15 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col1, _col5, _col7 - Statistics: Num rows: 1533288844555592 Data size: 284333973303297248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1533288844555592 Data size: 272067662546852480 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: int), _col5 (type: string), _col7 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1533288844555592 Data size: 284333973303297248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1533288844555592 Data size: 272067662546852480 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + - Statistics: Num rows: 1533288844555592 Data size: 284333973303297248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1533288844555592 Data size: 272067662546852480 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col1 (type: string), _col2 (type: string) Reducer 8 @@ -523,13 +458,13 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1533288844555592 Data size: 284333973303297248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1533288844555592 Data size: 272067662546852480 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 100 - Statistics: Num rows: 100 Data size: 18500 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 17700 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 100 Data size: 18500 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 100 Data size: 17700 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query10.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query10.q.out index 9df533b6f0..e076022ffa 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query10.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query10.q.out @@ -137,7 +137,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$4], sort4=[$6], sort5= HiveFilter(condition=[OR(IS NOT NULL($14), IS NOT NULL($16))]) HiveJoin(condition=[=($0, $17)], joinType=[left], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($0, $15)], joinType=[left], algorithm=[none], cost=[not available]) - HiveSemiJoin(condition=[=($0, $14)], joinType=[inner]) + HiveSemiJoin(condition=[=($0, $14)], joinType=[semi]) HiveJoin(condition=[=($5, $1)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($2, $3)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(c_customer_sk=[$0], c_current_cdemo_sk=[$2], c_current_addr_sk=[$4]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query14.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query14.q.out index dfb4c33864..69c49c0981 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query14.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query14.q.out @@ -231,7 +231,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC], dir1=[ HiveFilter(condition=[IS NOT NULL($3)]) HiveAggregate(group=[{0, 1, 2}], agg#0=[sum($3)], agg#1=[count()]) HiveProject(i_brand_id=[$1], i_class_id=[$2], i_category_id=[$3], $f3=[*(CAST($6):DECIMAL(10, 0), $7)]) - HiveSemiJoin(condition=[=($5, $9)], joinType=[inner]) + HiveSemiJoin(condition=[=($5, $9)], joinType=[semi]) HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11]) HiveFilter(condition=[IS NOT NULL($0)]) @@ -328,7 +328,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC], dir1=[ HiveFilter(condition=[IS NOT NULL($3)]) HiveAggregate(group=[{0, 1, 2}], agg#0=[sum($3)], agg#1=[count()]) HiveProject(i_brand_id=[$1], i_class_id=[$2], i_category_id=[$3], $f3=[*(CAST($6):DECIMAL(10, 0), $7)]) - HiveSemiJoin(condition=[=($5, $9)], joinType=[inner]) + HiveSemiJoin(condition=[=($5, $9)], joinType=[semi]) HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11]) HiveFilter(condition=[IS NOT NULL($0)]) @@ -425,7 +425,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC], dir1=[ HiveFilter(condition=[IS NOT NULL($3)]) HiveAggregate(group=[{0, 1, 2}], agg#0=[sum($3)], agg#1=[count()]) HiveProject(i_brand_id=[$1], i_class_id=[$2], i_category_id=[$3], $f3=[*(CAST($6):DECIMAL(10, 0), $7)]) - HiveSemiJoin(condition=[=($5, $9)], joinType=[inner]) + HiveSemiJoin(condition=[=($5, $9)], joinType=[semi]) HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11]) HiveFilter(condition=[IS NOT NULL($0)]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query16.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query16.q.out index 8126e432b2..1bfd9d8855 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query16.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query16.q.out @@ -74,7 +74,7 @@ CBO PLAN: HiveAggregate(group=[{}], agg#0=[count(DISTINCT $4)], agg#1=[sum($5)], agg#2=[sum($6)]) HiveFilter(condition=[IS NULL($13)]) HiveJoin(condition=[=($4, $14)], joinType=[left], algorithm=[none], cost=[not available]) - HiveSemiJoin(condition=[AND(<>($3, $13), =($4, $14))], joinType=[inner]) + HiveSemiJoin(condition=[AND(<>($3, $13), =($4, $14))], joinType=[semi]) HiveProject(cs_ship_date_sk=[$2], cs_ship_addr_sk=[$3], cs_call_center_sk=[$4], cs_warehouse_sk=[$5], cs_order_number=[$6], cs_ext_ship_cost=[$7], cs_net_profit=[$8], d_date_sk=[$9], d_date=[$10], ca_address_sk=[$0], ca_state=[$1], cc_call_center_sk=[$11], cc_county=[$12]) HiveJoin(condition=[=($4, $11)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($3, $0)], joinType=[inner], algorithm=[none], cost=[not available]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query23.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query23.q.out index 6aaf3a0c8d..4254e66a07 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query23.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query23.q.out @@ -121,7 +121,7 @@ HiveAggregate(group=[{}], agg#0=[sum($0)]) HiveProject(sales=[$0]) HiveUnion(all=[true]) HiveProject(sales=[*(CAST($4):DECIMAL(10, 0), $5)]) - HiveSemiJoin(condition=[=($2, $7)], joinType=[inner]) + HiveSemiJoin(condition=[=($2, $7)], joinType=[semi]) HiveJoin(condition=[=($3, $0)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(i_item_sk=[$0]) HiveAggregate(group=[{1}]) @@ -176,7 +176,7 @@ HiveAggregate(group=[{}], agg#0=[sum($0)]) HiveFilter(condition=[AND(IN($6, 1999, 2000, 2001, 2002), IS NOT NULL($0))]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) HiveProject(sales=[*(CAST($4):DECIMAL(10, 0), $5)]) - HiveSemiJoin(condition=[=($3, $7)], joinType=[inner]) + HiveSemiJoin(condition=[=($3, $7)], joinType=[semi]) HiveJoin(condition=[=($2, $0)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(i_item_sk=[$0]) HiveAggregate(group=[{1}]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query35.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query35.q.out index dcfc53be15..0eb3f7094c 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query35.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query35.q.out @@ -134,7 +134,7 @@ HiveProject(ca_state=[$0], cd_gender=[$1], cd_marital_status=[$2], cnt1=[$3], _o HiveFilter(condition=[OR(IS NOT NULL($11), IS NOT NULL($13))]) HiveJoin(condition=[=($0, $14)], joinType=[left], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($0, $12)], joinType=[left], algorithm=[none], cost=[not available]) - HiveSemiJoin(condition=[=($0, $11)], joinType=[inner]) + HiveSemiJoin(condition=[=($0, $11)], joinType=[semi]) HiveJoin(condition=[=($5, $1)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($2, $3)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(c_customer_sk=[$0], c_current_cdemo_sk=[$2], c_current_addr_sk=[$4]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query69.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query69.q.out index f0cca27177..4e617f99ed 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query69.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query69.q.out @@ -115,7 +115,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$4], sort4=[$6], dir0=[ HiveProject(c_customer_sk=[$0], c_current_cdemo_sk=[$1], c_current_addr_sk=[$2], ca_address_sk=[$3], ca_state=[$4], cd_demo_sk=[$5], cd_gender=[$6], cd_marital_status=[$7], cd_education_status=[$8], cd_purchase_estimate=[$9], cd_credit_rating=[$10], literalTrue=[$11], ws_bill_customer_sk0=[$12]) HiveFilter(condition=[IS NULL($11)]) HiveJoin(condition=[=($0, $12)], joinType=[left], algorithm=[none], cost=[not available]) - HiveSemiJoin(condition=[=($0, $11)], joinType=[inner]) + HiveSemiJoin(condition=[=($0, $11)], joinType=[semi]) HiveJoin(condition=[=($5, $1)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($2, $3)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(c_customer_sk=[$0], c_current_cdemo_sk=[$2], c_current_addr_sk=[$4]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query83.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query83.q.out index 5ee3bfc644..474193229a 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query83.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query83.q.out @@ -164,7 +164,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) HiveProject(d_date=[$0]) HiveAggregate(group=[{0}]) - HiveSemiJoin(condition=[=($1, $2)], joinType=[inner]) + HiveSemiJoin(condition=[=($1, $2)], joinType=[semi]) HiveProject(d_date=[$2], d_week_seq=[$4]) HiveFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($2))]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) @@ -188,7 +188,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) HiveProject(d_date=[$0]) HiveAggregate(group=[{0}]) - HiveSemiJoin(condition=[=($1, $2)], joinType=[inner]) + HiveSemiJoin(condition=[=($1, $2)], joinType=[semi]) HiveProject(d_date=[$2], d_week_seq=[$4]) HiveFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($2))]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) @@ -212,7 +212,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) HiveProject(d_date=[$0]) HiveAggregate(group=[{0}]) - HiveSemiJoin(condition=[=($1, $2)], joinType=[inner]) + HiveSemiJoin(condition=[=($1, $2)], joinType=[semi]) HiveProject(d_date=[$2], d_week_seq=[$4]) HiveFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($2))]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query94.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query94.q.out index 5cf486f480..b8521f4189 100644 --- a/ql/src/test/results/clientpositive/perf/tez/cbo_query94.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query94.q.out @@ -70,7 +70,7 @@ CBO PLAN: HiveAggregate(group=[{}], agg#0=[count(DISTINCT $4)], agg#1=[sum($5)], agg#2=[sum($6)]) HiveFilter(condition=[IS NULL($13)]) HiveJoin(condition=[=($4, $14)], joinType=[left], algorithm=[none], cost=[not available]) - HiveSemiJoin(condition=[AND(<>($3, $13), =($4, $14))], joinType=[inner]) + HiveSemiJoin(condition=[AND(<>($3, $13), =($4, $14))], joinType=[semi]) HiveProject(ws_ship_date_sk=[$2], ws_ship_addr_sk=[$3], ws_web_site_sk=[$4], ws_warehouse_sk=[$5], ws_order_number=[$6], ws_ext_ship_cost=[$7], ws_net_profit=[$8], d_date_sk=[$9], d_date=[$10], ca_address_sk=[$0], ca_state=[$1], web_site_sk=[$11], web_company_name=[$12]) HiveJoin(condition=[=($4, $11)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($3, $0)], joinType=[inner], algorithm=[none], cost=[not available]) diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query10.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query10.q.out index 2427cbd2ef..8633ba2e8e 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query10.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query10.q.out @@ -137,7 +137,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$4], sort4=[$6], sort5= HiveFilter(condition=[OR(IS NOT NULL($14), IS NOT NULL($16))]) HiveJoin(condition=[=($0, $17)], joinType=[left], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($0, $15)], joinType=[left], algorithm=[none], cost=[not available]) - HiveSemiJoin(condition=[=($0, $14)], joinType=[inner]) + HiveSemiJoin(condition=[=($0, $14)], joinType=[semi]) HiveJoin(condition=[=($5, $1)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($2, $3)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(c_customer_sk=[$0], c_current_cdemo_sk=[$2], c_current_addr_sk=[$4]) diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query16.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query16.q.out index 13c6477438..b40ca94ac6 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query16.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query16.q.out @@ -74,7 +74,7 @@ CBO PLAN: HiveAggregate(group=[{}], agg#0=[count(DISTINCT $4)], agg#1=[sum($5)], agg#2=[sum($6)]) HiveFilter(condition=[IS NULL($13)]) HiveJoin(condition=[=($4, $14)], joinType=[left], algorithm=[none], cost=[not available]) - HiveSemiJoin(condition=[AND(<>($3, $13), =($4, $14))], joinType=[inner]) + HiveSemiJoin(condition=[AND(<>($3, $13), =($4, $14))], joinType=[semi]) HiveProject(cs_ship_date_sk=[$2], cs_ship_addr_sk=[$3], cs_call_center_sk=[$4], cs_warehouse_sk=[$5], cs_order_number=[$6], cs_ext_ship_cost=[$7], cs_net_profit=[$8], d_date_sk=[$9], d_date=[$10], ca_address_sk=[$0], ca_state=[$1], cc_call_center_sk=[$11], cc_county=[$12]) HiveJoin(condition=[=($4, $11)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($3, $0)], joinType=[inner], algorithm=[none], cost=[not available]) diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query23.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query23.q.out index dfa794d1b8..d196ce69c8 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query23.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query23.q.out @@ -121,7 +121,7 @@ HiveAggregate(group=[{}], agg#0=[sum($0)]) HiveProject(sales=[$0]) HiveUnion(all=[true]) HiveProject(sales=[*(CAST($4):DECIMAL(10, 0), $5)]) - HiveSemiJoin(condition=[=($3, $7)], joinType=[inner]) + HiveSemiJoin(condition=[=($3, $7)], joinType=[semi]) HiveJoin(condition=[=($2, $0)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(ss_customer_sk=[$0]) HiveAggregate(group=[{0}]) @@ -167,7 +167,7 @@ HiveAggregate(group=[{}], agg#0=[sum($0)]) HiveProject(i_item_sk=[$0], substr=[substr($4, 1, 30)]) HiveTableScan(table=[[default, item]], table:alias=[item]) HiveProject(sales=[*(CAST($4):DECIMAL(10, 0), $5)]) - HiveSemiJoin(condition=[=($2, $7)], joinType=[inner]) + HiveSemiJoin(condition=[=($2, $7)], joinType=[semi]) HiveJoin(condition=[=($3, $0)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(ss_customer_sk=[$0]) HiveAggregate(group=[{0}]) diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query35.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query35.q.out index d3d6658280..2de681801d 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query35.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query35.q.out @@ -134,7 +134,7 @@ HiveProject(ca_state=[$0], cd_gender=[$1], cd_marital_status=[$2], cnt1=[$3], _o HiveFilter(condition=[OR(IS NOT NULL($11), IS NOT NULL($13))]) HiveJoin(condition=[=($0, $14)], joinType=[left], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($0, $12)], joinType=[left], algorithm=[none], cost=[not available]) - HiveSemiJoin(condition=[=($0, $11)], joinType=[inner]) + HiveSemiJoin(condition=[=($0, $11)], joinType=[semi]) HiveProject(c_customer_sk=[$0], c_current_cdemo_sk=[$1], c_current_addr_sk=[$2], ca_address_sk=[$9], ca_state=[$10], cd_demo_sk=[$3], cd_gender=[$4], cd_marital_status=[$5], cd_dep_count=[$6], cd_dep_employed_count=[$7], cd_dep_college_count=[$8]) HiveJoin(condition=[=($2, $9)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($3, $1)], joinType=[inner], algorithm=[none], cost=[not available]) diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query69.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query69.q.out index 5b739ce310..8cb93b4271 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query69.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query69.q.out @@ -115,7 +115,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$4], sort4=[$6], dir0=[ HiveProject(c_customer_sk=[$0], c_current_cdemo_sk=[$1], c_current_addr_sk=[$2], ca_address_sk=[$3], ca_state=[$4], cd_demo_sk=[$5], cd_gender=[$6], cd_marital_status=[$7], cd_education_status=[$8], cd_purchase_estimate=[$9], cd_credit_rating=[$10], literalTrue=[$11], ws_bill_customer_sk0=[$12]) HiveFilter(condition=[IS NULL($11)]) HiveJoin(condition=[=($0, $12)], joinType=[left], algorithm=[none], cost=[not available]) - HiveSemiJoin(condition=[=($0, $11)], joinType=[inner]) + HiveSemiJoin(condition=[=($0, $11)], joinType=[semi]) HiveJoin(condition=[=($5, $1)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($2, $3)], joinType=[inner], algorithm=[none], cost=[not available]) HiveProject(c_customer_sk=[$0], c_current_cdemo_sk=[$2], c_current_addr_sk=[$4]) diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query83.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query83.q.out index 6c3404d979..fe05a6e300 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query83.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query83.q.out @@ -160,7 +160,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) HiveProject(d_date=[$0]) HiveAggregate(group=[{0}]) - HiveSemiJoin(condition=[=($1, $2)], joinType=[inner]) + HiveSemiJoin(condition=[=($1, $2)], joinType=[semi]) HiveProject(d_date=[$2], d_week_seq=[$4]) HiveFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($2))]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) @@ -182,7 +182,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) HiveProject(d_date=[$0]) HiveAggregate(group=[{0}]) - HiveSemiJoin(condition=[=($1, $2)], joinType=[inner]) + HiveSemiJoin(condition=[=($1, $2)], joinType=[semi]) HiveProject(d_date=[$2], d_week_seq=[$4]) HiveFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($2))]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) @@ -204,7 +204,7 @@ HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[100]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) HiveProject(d_date=[$0]) HiveAggregate(group=[{0}]) - HiveSemiJoin(condition=[=($1, $2)], joinType=[inner]) + HiveSemiJoin(condition=[=($1, $2)], joinType=[semi]) HiveProject(d_date=[$2], d_week_seq=[$4]) HiveFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($2))]) HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim]) diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query94.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query94.q.out index 8efda1ebd2..ef108a6578 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query94.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query94.q.out @@ -70,7 +70,7 @@ CBO PLAN: HiveAggregate(group=[{}], agg#0=[count(DISTINCT $4)], agg#1=[sum($5)], agg#2=[sum($6)]) HiveFilter(condition=[IS NULL($13)]) HiveJoin(condition=[=($4, $14)], joinType=[left], algorithm=[none], cost=[not available]) - HiveSemiJoin(condition=[AND(<>($3, $13), =($4, $14))], joinType=[inner]) + HiveSemiJoin(condition=[AND(<>($3, $13), =($4, $14))], joinType=[semi]) HiveProject(ws_ship_date_sk=[$2], ws_ship_addr_sk=[$3], ws_web_site_sk=[$4], ws_warehouse_sk=[$5], ws_order_number=[$6], ws_ext_ship_cost=[$7], ws_net_profit=[$8], d_date_sk=[$9], d_date=[$10], ca_address_sk=[$0], ca_state=[$1], web_site_sk=[$11], web_company_name=[$12]) HiveJoin(condition=[=($4, $11)], joinType=[inner], algorithm=[none], cost=[not available]) HiveJoin(condition=[=($3, $0)], joinType=[inner], algorithm=[none], cost=[not available]) diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/mv_query44.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/mv_query44.q.out index f56f9889b7..1c794c5011 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/mv_query44.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/mv_query44.q.out @@ -19,8 +19,7 @@ POSTHOOK: type: CREATE_MATERIALIZED_VIEW POSTHOOK: Input: default@store_sales POSTHOOK: Output: database:default POSTHOOK: Output: default@mv_store_sales_item_customer -Warning: Shuffle Join MERGEJOIN[153][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[154][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[109][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * @@ -98,155 +97,124 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 5 (SIMPLE_EDGE) -Reducer 13 <- Map 12 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE) +Reducer 11 <- Map 10 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) -Reducer 4 <- Reducer 13 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Reducer 15 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 16 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 16 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 3 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) +Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) +Reducer 6 <- Map 12 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) +Reducer 7 <- Map 12 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Reducer 9 <- Reducer 3 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 10 vectorized - File Output Operator [FS_204] - Limit [LIM_203] (rows=100 width=218) + Reducer 8 vectorized + File Output Operator [FS_146] + Limit [LIM_145] (rows=100 width=218) Number of rows:100 - Select Operator [SEL_202] (rows=6951 width=218) + Select Operator [SEL_144] (rows=6951 width=218) Output:["_col0","_col1","_col2"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_106] - Select Operator [SEL_105] (rows=6951 width=218) + <-Reducer 7 [SIMPLE_EDGE] + SHUFFLE [RS_70] + Select Operator [SEL_69] (rows=6951 width=218) Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_159] (rows=6951 width=218) - Conds:RS_102._col2=RS_201._col0(Inner),Output:["_col1","_col5","_col7"] - <-Map 16 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_201] + Merge Join Operator [MERGEJOIN_113] (rows=6951 width=218) + Conds:RS_66._col2=RS_143._col0(Inner),Output:["_col1","_col5","_col7"] + <-Map 12 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_143] PartitionCols:_col0 - Select Operator [SEL_199] (rows=462000 width=111) + Select Operator [SEL_141] (rows=462000 width=111) Output:["_col0","_col1"] - TableScan [TS_92] (rows=462000 width=111) + TableScan [TS_56] (rows=462000 width=111) default@item,i1,Tbl:COMPLETE,Col:COMPLETE,Output:["i_item_sk","i_product_name"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_102] + <-Reducer 6 [SIMPLE_EDGE] + SHUFFLE [RS_66] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_158] (rows=6951 width=115) - Conds:RS_99._col0=RS_200._col0(Inner),Output:["_col1","_col2","_col5"] - <-Map 16 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_200] + Merge Join Operator [MERGEJOIN_112] (rows=6951 width=115) + Conds:RS_63._col0=RS_142._col0(Inner),Output:["_col1","_col2","_col5"] + <-Map 12 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_142] PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_199] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_99] + Please refer to the previous Select Operator [SEL_141] + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_63] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_157] (rows=6951 width=12) - Conds:RS_193._col1=RS_198._col1(Inner),Output:["_col0","_col1","_col2"] - <-Reducer 11 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_198] + Merge Join Operator [MERGEJOIN_111] (rows=6951 width=12) + Conds:RS_135._col1=RS_140._col1(Inner),Output:["_col0","_col1","_col2"] + <-Reducer 4 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_135] PartitionCols:_col1 - Select Operator [SEL_197] (rows=6951 width=8) + Select Operator [SEL_134] (rows=6951 width=8) Output:["_col0","_col1"] - Filter Operator [FIL_196] (rows=6951 width=116) + Filter Operator [FIL_133] (rows=6951 width=116) predicate:(rank_window_0 < 11) - PTF Operator [PTF_195] (rows=20854 width=116) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 DESC NULLS LAST","partition by:":"0"}] - Select Operator [SEL_194] (rows=20854 width=116) - Output:["_col2","_col3"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_85] + PTF Operator [PTF_132] (rows=20854 width=116) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"0"}] + Select Operator [SEL_131] (rows=20854 width=116) + Output:["_col0","_col1"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_21] PartitionCols:0 - Filter Operator [FIL_38] (rows=20854 width=228) - predicate:(_col3 > _col1) - Merge Join Operator [MERGEJOIN_154] (rows=62562 width=228) - Conds:(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 15 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_188] - Select Operator [SEL_187] (rows=62562 width=116) + Filter Operator [FIL_20] (rows=20854 width=228) + predicate:(_col1 > (0.9 * _col2)) + Merge Join Operator [MERGEJOIN_109] (rows=62562 width=228) + Conds:(Inner),Output:["_col0","_col1","_col2"] + <-Reducer 11 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_130] + Select Operator [SEL_129] (rows=1 width=112) + Output:["_col0"] + Filter Operator [FIL_128] (rows=1 width=120) + predicate:(_col1 is not null and _col2 is not null) + Select Operator [SEL_127] (rows=1 width=120) + Output:["_col1","_col2"] + Group By Operator [GBY_126] (rows=1 width=124) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 + <-Map 10 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_125] + PartitionCols:_col0 + Group By Operator [GBY_124] (rows=258 width=124) + Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","count(_col1)"],keys:true + Select Operator [SEL_123] (rows=287946 width=114) + Output:["_col1"] + Filter Operator [FIL_122] (rows=287946 width=114) + predicate:(ss_hdemo_sk is null and (ss_store_sk = 410)) + TableScan [TS_8] (rows=575995635 width=114) + default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_hdemo_sk","ss_store_sk","ss_net_profit"] + <-Reducer 2 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_121] + Select Operator [SEL_120] (rows=62562 width=116) Output:["_col0","_col1"] - Filter Operator [FIL_186] (rows=62562 width=124) + Filter Operator [FIL_119] (rows=62562 width=124) predicate:(_col1 is not null and _col2 is not null) - Group By Operator [GBY_185] (rows=62562 width=124) + Group By Operator [GBY_118] (rows=62562 width=124) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Map 14 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_184] + <-Map 1 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_117] PartitionCols:_col0 - Group By Operator [GBY_183] (rows=3199976 width=124) + Group By Operator [GBY_116] (rows=3199976 width=124) Output:["_col0","_col1","_col2"],aggregations:["sum(ss_net_profit)","count(ss_net_profit)"],keys:ss_item_sk - Select Operator [SEL_182] (rows=6399952 width=114) + Select Operator [SEL_115] (rows=6399952 width=114) Output:["ss_item_sk","ss_net_profit"] - Filter Operator [FIL_181] (rows=6399952 width=114) + Filter Operator [FIL_114] (rows=6399952 width=114) predicate:(ss_store_sk = 410) - TableScan [TS_24] (rows=575995635 width=114) + TableScan [TS_0] (rows=575995635 width=114) default@store_sales,ss1,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_item_sk","ss_store_sk","ss_net_profit"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_35] - Merge Join Operator [MERGEJOIN_153] (rows=1 width=112) - Conds:(Inner),Output:["_col1"] - <-Reducer 13 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_180] - Select Operator [SEL_179] (rows=1 width=112) - Output:["_col0"] - Filter Operator [FIL_178] (rows=1 width=120) - predicate:(_col1 is not null and _col2 is not null) - Select Operator [SEL_177] (rows=1 width=120) - Output:["_col1","_col2"] - Group By Operator [GBY_176] (rows=1 width=124) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Map 12 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_175] - PartitionCols:_col0 - Group By Operator [GBY_174] (rows=258 width=124) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","count(_col1)"],keys:true - Select Operator [SEL_173] (rows=287946 width=114) - Output:["_col1"] - Filter Operator [FIL_172] (rows=287946 width=114) - predicate:(ss_hdemo_sk is null and (ss_store_sk = 410)) - TableScan [TS_15] (rows=575995635 width=114) - default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_hdemo_sk","ss_store_sk","ss_net_profit"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_171] - Select Operator [SEL_170] (rows=1 width=8) - Filter Operator [FIL_169] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_168] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_167] - Group By Operator [GBY_166] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_165] (rows=1 width=4) - Group By Operator [GBY_164] (rows=1 width=4) - Output:["_col0"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_163] - PartitionCols:_col0 - Group By Operator [GBY_162] (rows=18 width=4) - Output:["_col0"],keys:true - Select Operator [SEL_161] (rows=287946 width=7) - Filter Operator [FIL_160] (rows=287946 width=7) - predicate:(ss_hdemo_sk is null and (ss_store_sk = 410)) - TableScan [TS_0] (rows=575995635 width=7) - default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_hdemo_sk","ss_store_sk"] - <-Reducer 6 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_193] + <-Reducer 9 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_140] PartitionCols:_col1 - Select Operator [SEL_192] (rows=6951 width=8) + Select Operator [SEL_139] (rows=6951 width=8) Output:["_col0","_col1"] - Filter Operator [FIL_191] (rows=6951 width=116) + Filter Operator [FIL_138] (rows=6951 width=116) predicate:(rank_window_0 < 11) - PTF Operator [PTF_190] (rows=20854 width=116) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"0"}] - Select Operator [SEL_189] (rows=20854 width=116) - Output:["_col2","_col3"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_39] + PTF Operator [PTF_137] (rows=20854 width=116) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 DESC NULLS LAST","partition by:":"0"}] + Select Operator [SEL_136] (rows=20854 width=116) + Output:["_col0","_col1"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_49] PartitionCols:0 - Please refer to the previous Filter Operator [FIL_38] + Please refer to the previous Filter Operator [FIL_20] diff --git a/ql/src/test/results/clientpositive/perf/tez/constraints/query44.q.out b/ql/src/test/results/clientpositive/perf/tez/constraints/query44.q.out index 80a9cdc356..b2c012382b 100644 --- a/ql/src/test/results/clientpositive/perf/tez/constraints/query44.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/constraints/query44.q.out @@ -1,5 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[153][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product -Warning: Shuffle Join MERGEJOIN[154][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[109][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * @@ -77,155 +76,124 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 5 (SIMPLE_EDGE) -Reducer 13 <- Map 12 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE) +Reducer 11 <- Map 10 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) -Reducer 4 <- Reducer 13 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Reducer 15 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 16 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 16 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 3 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) +Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) +Reducer 6 <- Map 12 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) +Reducer 7 <- Map 12 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Reducer 9 <- Reducer 3 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 - Reducer 10 vectorized - File Output Operator [FS_204] - Limit [LIM_203] (rows=100 width=218) + Reducer 8 vectorized + File Output Operator [FS_146] + Limit [LIM_145] (rows=100 width=218) Number of rows:100 - Select Operator [SEL_202] (rows=6951 width=218) + Select Operator [SEL_144] (rows=6951 width=218) Output:["_col0","_col1","_col2"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_106] - Select Operator [SEL_105] (rows=6951 width=218) + <-Reducer 7 [SIMPLE_EDGE] + SHUFFLE [RS_70] + Select Operator [SEL_69] (rows=6951 width=218) Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_159] (rows=6951 width=218) - Conds:RS_102._col2=RS_201._col0(Inner),Output:["_col1","_col5","_col7"] - <-Map 16 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_201] + Merge Join Operator [MERGEJOIN_113] (rows=6951 width=218) + Conds:RS_66._col2=RS_143._col0(Inner),Output:["_col1","_col5","_col7"] + <-Map 12 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_143] PartitionCols:_col0 - Select Operator [SEL_199] (rows=462000 width=111) + Select Operator [SEL_141] (rows=462000 width=111) Output:["_col0","_col1"] - TableScan [TS_92] (rows=462000 width=111) + TableScan [TS_56] (rows=462000 width=111) default@item,i1,Tbl:COMPLETE,Col:COMPLETE,Output:["i_item_sk","i_product_name"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_102] + <-Reducer 6 [SIMPLE_EDGE] + SHUFFLE [RS_66] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_158] (rows=6951 width=115) - Conds:RS_99._col0=RS_200._col0(Inner),Output:["_col1","_col2","_col5"] - <-Map 16 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_200] + Merge Join Operator [MERGEJOIN_112] (rows=6951 width=115) + Conds:RS_63._col0=RS_142._col0(Inner),Output:["_col1","_col2","_col5"] + <-Map 12 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_142] PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_199] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_99] + Please refer to the previous Select Operator [SEL_141] + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_63] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_157] (rows=6951 width=12) - Conds:RS_193._col1=RS_198._col1(Inner),Output:["_col0","_col1","_col2"] - <-Reducer 11 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_198] + Merge Join Operator [MERGEJOIN_111] (rows=6951 width=12) + Conds:RS_135._col1=RS_140._col1(Inner),Output:["_col0","_col1","_col2"] + <-Reducer 4 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_135] PartitionCols:_col1 - Select Operator [SEL_197] (rows=6951 width=8) + Select Operator [SEL_134] (rows=6951 width=8) Output:["_col0","_col1"] - Filter Operator [FIL_196] (rows=6951 width=116) + Filter Operator [FIL_133] (rows=6951 width=116) predicate:(rank_window_0 < 11) - PTF Operator [PTF_195] (rows=20854 width=116) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 DESC NULLS LAST","partition by:":"0"}] - Select Operator [SEL_194] (rows=20854 width=116) - Output:["_col2","_col3"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_85] + PTF Operator [PTF_132] (rows=20854 width=116) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"0"}] + Select Operator [SEL_131] (rows=20854 width=116) + Output:["_col0","_col1"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_21] PartitionCols:0 - Filter Operator [FIL_38] (rows=20854 width=228) - predicate:(_col3 > _col1) - Merge Join Operator [MERGEJOIN_154] (rows=62562 width=228) - Conds:(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 15 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_188] - Select Operator [SEL_187] (rows=62562 width=116) + Filter Operator [FIL_20] (rows=20854 width=228) + predicate:(_col1 > (0.9 * _col2)) + Merge Join Operator [MERGEJOIN_109] (rows=62562 width=228) + Conds:(Inner),Output:["_col0","_col1","_col2"] + <-Reducer 11 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_130] + Select Operator [SEL_129] (rows=1 width=112) + Output:["_col0"] + Filter Operator [FIL_128] (rows=1 width=120) + predicate:(_col1 is not null and _col2 is not null) + Select Operator [SEL_127] (rows=1 width=120) + Output:["_col1","_col2"] + Group By Operator [GBY_126] (rows=1 width=124) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 + <-Map 10 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_125] + PartitionCols:_col0 + Group By Operator [GBY_124] (rows=258 width=124) + Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","count(_col1)"],keys:true + Select Operator [SEL_123] (rows=287946 width=114) + Output:["_col1"] + Filter Operator [FIL_122] (rows=287946 width=114) + predicate:(ss_hdemo_sk is null and (ss_store_sk = 410)) + TableScan [TS_8] (rows=575995635 width=114) + default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_hdemo_sk","ss_store_sk","ss_net_profit"] + <-Reducer 2 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_121] + Select Operator [SEL_120] (rows=62562 width=116) Output:["_col0","_col1"] - Filter Operator [FIL_186] (rows=62562 width=124) + Filter Operator [FIL_119] (rows=62562 width=124) predicate:(_col1 is not null and _col2 is not null) - Group By Operator [GBY_185] (rows=62562 width=124) + Group By Operator [GBY_118] (rows=62562 width=124) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Map 14 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_184] + <-Map 1 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_117] PartitionCols:_col0 - Group By Operator [GBY_183] (rows=3199976 width=124) + Group By Operator [GBY_116] (rows=3199976 width=124) Output:["_col0","_col1","_col2"],aggregations:["sum(ss_net_profit)","count(ss_net_profit)"],keys:ss_item_sk - Select Operator [SEL_182] (rows=6399952 width=114) + Select Operator [SEL_115] (rows=6399952 width=114) Output:["ss_item_sk","ss_net_profit"] - Filter Operator [FIL_181] (rows=6399952 width=114) + Filter Operator [FIL_114] (rows=6399952 width=114) predicate:(ss_store_sk = 410) - TableScan [TS_24] (rows=575995635 width=114) + TableScan [TS_0] (rows=575995635 width=114) default@store_sales,ss1,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_item_sk","ss_store_sk","ss_net_profit"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_35] - Merge Join Operator [MERGEJOIN_153] (rows=1 width=112) - Conds:(Inner),Output:["_col1"] - <-Reducer 13 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_180] - Select Operator [SEL_179] (rows=1 width=112) - Output:["_col0"] - Filter Operator [FIL_178] (rows=1 width=120) - predicate:(_col1 is not null and _col2 is not null) - Select Operator [SEL_177] (rows=1 width=120) - Output:["_col1","_col2"] - Group By Operator [GBY_176] (rows=1 width=124) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Map 12 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_175] - PartitionCols:_col0 - Group By Operator [GBY_174] (rows=258 width=124) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","count(_col1)"],keys:true - Select Operator [SEL_173] (rows=287946 width=114) - Output:["_col1"] - Filter Operator [FIL_172] (rows=287946 width=114) - predicate:(ss_hdemo_sk is null and (ss_store_sk = 410)) - TableScan [TS_15] (rows=575995635 width=114) - default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_hdemo_sk","ss_store_sk","ss_net_profit"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_171] - Select Operator [SEL_170] (rows=1 width=8) - Filter Operator [FIL_169] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_168] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 2 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_167] - Group By Operator [GBY_166] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_165] (rows=1 width=4) - Group By Operator [GBY_164] (rows=1 width=4) - Output:["_col0"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_163] - PartitionCols:_col0 - Group By Operator [GBY_162] (rows=18 width=4) - Output:["_col0"],keys:true - Select Operator [SEL_161] (rows=287946 width=7) - Filter Operator [FIL_160] (rows=287946 width=7) - predicate:(ss_hdemo_sk is null and (ss_store_sk = 410)) - TableScan [TS_0] (rows=575995635 width=7) - default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_hdemo_sk","ss_store_sk"] - <-Reducer 6 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_193] + <-Reducer 9 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_140] PartitionCols:_col1 - Select Operator [SEL_192] (rows=6951 width=8) + Select Operator [SEL_139] (rows=6951 width=8) Output:["_col0","_col1"] - Filter Operator [FIL_191] (rows=6951 width=116) + Filter Operator [FIL_138] (rows=6951 width=116) predicate:(rank_window_0 < 11) - PTF Operator [PTF_190] (rows=20854 width=116) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"0"}] - Select Operator [SEL_189] (rows=20854 width=116) - Output:["_col2","_col3"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_39] + PTF Operator [PTF_137] (rows=20854 width=116) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 DESC NULLS LAST","partition by:":"0"}] + Select Operator [SEL_136] (rows=20854 width=116) + Output:["_col0","_col1"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_49] PartitionCols:0 - Please refer to the previous Filter Operator [FIL_38] + Please refer to the previous Filter Operator [FIL_20] diff --git a/ql/src/test/results/clientpositive/perf/tez/query44.q.out b/ql/src/test/results/clientpositive/perf/tez/query44.q.out index 0017e66090..11ed535275 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query44.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query44.q.out @@ -1,5 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[151][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 9' is a cross product -Warning: Shuffle Join MERGEJOIN[152][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 10' is a cross product +Warning: Shuffle Join MERGEJOIN[109][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 8' is a cross product PREHOOK: query: explain select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * @@ -77,157 +76,126 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 16 (CUSTOM_SIMPLE_EDGE), Reducer 9 (CUSTOM_SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 10 (SIMPLE_EDGE) -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) +Reducer 10 <- Reducer 8 (SIMPLE_EDGE) +Reducer 12 <- Map 11 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 1 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) +Reducer 5 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) -Reducer 9 <- Reducer 14 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) +Reducer 8 <- Reducer 12 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) +Reducer 9 <- Reducer 8 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:100 Stage-1 Reducer 4 vectorized - File Output Operator [FS_203] - Limit [LIM_202] (rows=100 width=218) + File Output Operator [FS_147] + Limit [LIM_146] (rows=100 width=218) Number of rows:100 - Select Operator [SEL_201] (rows=6951 width=218) + Select Operator [SEL_145] (rows=6951 width=218) Output:["_col0","_col1","_col2"] <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_109] - Select Operator [SEL_108] (rows=6951 width=218) + SHUFFLE [RS_73] + Select Operator [SEL_72] (rows=6951 width=218) Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_157] (rows=6951 width=218) - Conds:RS_105._col3=RS_106._col3(Inner),Output:["_col1","_col3","_col5"] + Merge Join Operator [MERGEJOIN_113] (rows=6951 width=218) + Conds:RS_69._col3=RS_70._col3(Inner),Output:["_col1","_col3","_col5"] <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_105] + SHUFFLE [RS_69] PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_153] (rows=6951 width=111) - Conds:RS_160._col0=RS_195._col0(Inner),Output:["_col1","_col3"] + Merge Join Operator [MERGEJOIN_110] (rows=6951 width=111) + Conds:RS_116._col0=RS_139._col0(Inner),Output:["_col1","_col3"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_160] + SHUFFLE [RS_116] PartitionCols:_col0 - Select Operator [SEL_159] (rows=462000 width=111) + Select Operator [SEL_115] (rows=462000 width=111) Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=462000 width=111) + Filter Operator [FIL_114] (rows=462000 width=111) predicate:i_item_sk is not null TableScan [TS_0] (rows=462000 width=111) default@item,i1,Tbl:COMPLETE,Col:COMPLETE,Output:["i_item_sk","i_product_name"] - <-Reducer 11 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_195] + <-Reducer 9 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_139] PartitionCols:_col0 - Select Operator [SEL_194] (rows=6951 width=8) + Select Operator [SEL_138] (rows=6951 width=8) Output:["_col0","_col1"] - Filter Operator [FIL_193] (rows=6951 width=116) - predicate:((rank_window_0 < 11) and _col2 is not null) - PTF Operator [PTF_192] (rows=20854 width=116) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST","partition by:":"0"}] - Select Operator [SEL_191] (rows=20854 width=116) - Output:["_col2","_col3"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_42] + Filter Operator [FIL_137] (rows=6951 width=116) + predicate:((rank_window_0 < 11) and _col0 is not null) + PTF Operator [PTF_136] (rows=20854 width=116) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"0"}] + Select Operator [SEL_135] (rows=20854 width=116) + Output:["_col0","_col1"] + <-Reducer 8 [SIMPLE_EDGE] + SHUFFLE [RS_24] PartitionCols:0 - Filter Operator [FIL_41] (rows=20854 width=228) - predicate:(_col3 > _col1) - Merge Join Operator [MERGEJOIN_152] (rows=62562 width=228) - Conds:(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 16 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_190] - Select Operator [SEL_189] (rows=62562 width=116) + Filter Operator [FIL_23] (rows=20854 width=228) + predicate:(_col1 > (0.9 * _col2)) + Merge Join Operator [MERGEJOIN_109] (rows=62562 width=228) + Conds:(Inner),Output:["_col0","_col1","_col2"] + <-Reducer 12 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_134] + Select Operator [SEL_133] (rows=1 width=112) + Output:["_col0"] + Filter Operator [FIL_132] (rows=1 width=120) + predicate:(_col1 is not null and _col2 is not null) + Select Operator [SEL_131] (rows=1 width=120) + Output:["_col1","_col2"] + Group By Operator [GBY_130] (rows=1 width=124) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 + <-Map 11 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_129] + PartitionCols:_col0 + Group By Operator [GBY_128] (rows=258 width=124) + Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","count(_col1)"],keys:true + Select Operator [SEL_127] (rows=287946 width=114) + Output:["_col1"] + Filter Operator [FIL_126] (rows=287946 width=114) + predicate:(ss_hdemo_sk is null and (ss_store_sk = 410)) + TableScan [TS_11] (rows=575995635 width=114) + default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_hdemo_sk","ss_store_sk","ss_net_profit"] + <-Reducer 7 [CUSTOM_SIMPLE_EDGE] vectorized + PARTITION_ONLY_SHUFFLE [RS_125] + Select Operator [SEL_124] (rows=62562 width=116) Output:["_col0","_col1"] - Filter Operator [FIL_188] (rows=62562 width=124) + Filter Operator [FIL_123] (rows=62562 width=124) predicate:(_col1 is not null and _col2 is not null) - Group By Operator [GBY_187] (rows=62562 width=124) + Group By Operator [GBY_122] (rows=62562 width=124) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Map 15 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_186] + <-Map 6 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_121] PartitionCols:_col0 - Group By Operator [GBY_185] (rows=3199976 width=124) + Group By Operator [GBY_120] (rows=3199976 width=124) Output:["_col0","_col1","_col2"],aggregations:["sum(ss_net_profit)","count(ss_net_profit)"],keys:ss_item_sk - Select Operator [SEL_184] (rows=6399952 width=114) + Select Operator [SEL_119] (rows=6399952 width=114) Output:["ss_item_sk","ss_net_profit"] - Filter Operator [FIL_183] (rows=6399952 width=114) + Filter Operator [FIL_118] (rows=6399952 width=114) predicate:(ss_store_sk = 410) - TableScan [TS_27] (rows=575995635 width=114) + TableScan [TS_3] (rows=575995635 width=114) default@store_sales,ss1,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_item_sk","ss_store_sk","ss_net_profit"] - <-Reducer 9 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_38] - Merge Join Operator [MERGEJOIN_151] (rows=1 width=112) - Conds:(Inner),Output:["_col1"] - <-Reducer 14 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_182] - Select Operator [SEL_181] (rows=1 width=112) - Output:["_col0"] - Filter Operator [FIL_180] (rows=1 width=120) - predicate:(_col1 is not null and _col2 is not null) - Select Operator [SEL_179] (rows=1 width=120) - Output:["_col1","_col2"] - Group By Operator [GBY_178] (rows=1 width=124) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_177] - PartitionCols:_col0 - Group By Operator [GBY_176] (rows=258 width=124) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","count(_col1)"],keys:true - Select Operator [SEL_175] (rows=287946 width=114) - Output:["_col1"] - Filter Operator [FIL_174] (rows=287946 width=114) - predicate:(ss_hdemo_sk is null and (ss_store_sk = 410)) - TableScan [TS_18] (rows=575995635 width=114) - default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_hdemo_sk","ss_store_sk","ss_net_profit"] - <-Reducer 8 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_173] - Select Operator [SEL_172] (rows=1 width=8) - Filter Operator [FIL_171] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_170] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 7 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_169] - Group By Operator [GBY_168] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_167] (rows=1 width=4) - Group By Operator [GBY_166] (rows=1 width=4) - Output:["_col0"],keys:KEY._col0 - <-Map 6 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_165] - PartitionCols:_col0 - Group By Operator [GBY_164] (rows=18 width=4) - Output:["_col0"],keys:true - Select Operator [SEL_163] (rows=287946 width=7) - Filter Operator [FIL_162] (rows=287946 width=7) - predicate:(ss_hdemo_sk is null and (ss_store_sk = 410)) - TableScan [TS_3] (rows=575995635 width=7) - default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_hdemo_sk","ss_store_sk"] <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_106] + SHUFFLE [RS_70] PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_156] (rows=6951 width=111) - Conds:RS_161._col0=RS_200._col0(Inner),Output:["_col1","_col3"] + Merge Join Operator [MERGEJOIN_112] (rows=6951 width=111) + Conds:RS_117._col0=RS_144._col0(Inner),Output:["_col1","_col3"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_161] + SHUFFLE [RS_117] PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_159] - <-Reducer 12 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_200] + Please refer to the previous Select Operator [SEL_115] + <-Reducer 10 [SIMPLE_EDGE] vectorized + SHUFFLE [RS_144] PartitionCols:_col0 - Select Operator [SEL_199] (rows=6951 width=8) + Select Operator [SEL_143] (rows=6951 width=8) Output:["_col0","_col1"] - Filter Operator [FIL_198] (rows=6951 width=116) - predicate:((rank_window_0 < 11) and _col2 is not null) - PTF Operator [PTF_197] (rows=20854 width=116) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 DESC NULLS LAST","partition by:":"0"}] - Select Operator [SEL_196] (rows=20854 width=116) - Output:["_col2","_col3"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_91] + Filter Operator [FIL_142] (rows=6951 width=116) + predicate:((rank_window_0 < 11) and _col0 is not null) + PTF Operator [PTF_141] (rows=20854 width=116) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 DESC NULLS LAST","partition by:":"0"}] + Select Operator [SEL_140] (rows=20854 width=116) + Output:["_col0","_col1"] + <-Reducer 8 [SIMPLE_EDGE] + SHUFFLE [RS_55] PartitionCols:0 - Please refer to the previous Filter Operator [FIL_41] + Please refer to the previous Filter Operator [FIL_23] diff --git a/ql/src/test/results/clientpositive/plan_json.q.out b/ql/src/test/results/clientpositive/plan_json.q.out index c447283eb0..65e2220219 100644 --- a/ql/src/test/results/clientpositive/plan_json.q.out +++ b/ql/src/test/results/clientpositive/plan_json.q.out @@ -6,4 +6,4 @@ POSTHOOK: query: EXPLAIN FORMATTED SELECT count(1) FROM src POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 0.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"key\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"key\",\n \"ndv\": 316\n },\n {\n \"name\": \"value\",\n \"ndv\": 307\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate\",\n \"group\": [],\n \"aggs\": [\n {\n \"agg\": \"count\",\n \"type\": {\n \"type\": \"BIGINT\",\n \"nullable\": true\n },\n \"distinct\": false,\n \"operands\": []\n }\n ],\n \"rowCount\": 1.0\n }\n ]\n}","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_7"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"src\"\n ],\n \"table:alias\": \"src\",\n \"inputs\": [],\n \"rowCount\": 500.0,\n \"avgRowSize\": 0.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"key\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"key\",\n \"ndv\": 316\n },\n {\n \"name\": \"value\",\n \"ndv\": 307\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate\",\n \"group\": [],\n \"aggs\": [\n {\n \"agg\": {\n \"name\": \"count\",\n \"kind\": \"COUNT\",\n \"syntax\": \"FUNCTION_STAR\"\n },\n \"type\": {\n \"type\": \"BIGINT\",\n \"nullable\": true\n },\n \"distinct\": false,\n \"operands\": [],\n \"name\": null\n }\n ],\n \"rowCount\": 1.0\n }\n ]\n}","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_7"}}}}}} diff --git a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out index 693198e914..33fb89e0a0 100644 --- a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out @@ -783,12 +783,52 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out b/ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out index 502fe65b40..690c3b976b 100644 --- a/ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out +++ b/ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out @@ -690,12 +690,88 @@ PLAN VECTORIZATION: enabledConditionsMet: [hive.vectorized.execution.enabled IS true] STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesparquet + Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 5] + Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: tinyint), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterConstantBooleanVectorExpression(val 0) + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/spark/semijoin.q.out b/ql/src/test/results/clientpositive/spark/semijoin.q.out index c42332d964..b6d6559c30 100644 --- a/ql/src/test/results/clientpositive/spark/semijoin.q.out +++ b/ql/src/test/results/clientpositive/spark/semijoin.q.out @@ -2955,7 +2955,7 @@ POSTHOOK: Input: default@part #### A masked pattern was here #### CBO PLAN: HiveProject(p_partkey=[$0]) - HiveSemiJoin(condition=[=($1, $2)], joinType=[inner]) + HiveSemiJoin(condition=[=($1, $2)], joinType=[semi]) HiveProject(p_partkey=[$0], p_name=[$1]) HiveFilter(condition=[IS NOT NULL($1)]) HiveTableScan(table=[[default, part]], table:alias=[pp]) @@ -3017,7 +3017,7 @@ POSTHOOK: Input: default@part #### A masked pattern was here #### CBO PLAN: HiveAggregate(group=[{}], agg#0=[count()]) - HiveSemiJoin(condition=[=($0, $1)], joinType=[inner]) + HiveSemiJoin(condition=[=($0, $1)], joinType=[semi]) HiveProject(p_partkey=[$0]) HiveFilter(condition=[IS NOT NULL($0)]) HiveTableScan(table=[[default, part]], table:alias=[pp]) diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out index 016e97ffdd..5be59584fc 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out @@ -2745,7 +2745,7 @@ STAGE PLANS: Reduce Operator Tree: Join Operator condition map: - Left Semi Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col1 (type: bigint) 1 _col0 (type: bigint) @@ -2778,17 +2778,11 @@ STAGE PLANS: expressions: _col1 (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -2971,7 +2965,7 @@ STAGE PLANS: Reduce Operator Tree: Join Operator condition map: - Left Semi Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col1 (type: bigint) 1 _col0 (type: bigint) @@ -3004,17 +2998,11 @@ STAGE PLANS: expressions: _col1 (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/spark/subquery_multi.q.out b/ql/src/test/results/clientpositive/spark/subquery_multi.q.out index ad95d2f127..de871684c9 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_multi.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_multi.q.out @@ -1725,7 +1725,7 @@ HiveProject(p_partkey=[$0], p_name=[$1], p_mfgr=[$2], p_brand=[$3], p_type=[$4], HiveTableScan(table=[[default, part_null]], table:alias=[part_null]) HiveProject(p_type=[$0], c=[$1], ck=[$2]) HiveAggregate(group=[{1}], c=[COUNT()], ck=[COUNT($2)]) - HiveSemiJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[inner]) + HiveSemiJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[semi]) HiveProject(p_brand=[$3], p_type=[$4], p_container=[$6]) HiveFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) HiveTableScan(table=[[default, part]], table:alias=[part]) @@ -1734,7 +1734,7 @@ HiveProject(p_partkey=[$0], p_name=[$1], p_mfgr=[$2], p_brand=[$3], p_type=[$4], HiveTableScan(table=[[default, part]], table:alias=[pp]) HiveProject(p_container=[$1], literalTrue=[true], p_type=[$0]) HiveAggregate(group=[{1, 2}]) - HiveSemiJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[inner]) + HiveSemiJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[semi]) HiveProject(p_brand=[$3], p_type=[$4], p_container=[$6]) HiveFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3), IS NOT NULL($6))]) HiveTableScan(table=[[default, part]], table:alias=[part]) diff --git a/ql/src/test/results/clientpositive/spark/subquery_notin.q.out b/ql/src/test/results/clientpositive/spark/subquery_notin.q.out index 38284482de..7c13e3a43f 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_notin.q.out @@ -3739,17 +3739,11 @@ STAGE PLANS: expressions: _col1 (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -3820,7 +3814,7 @@ STAGE PLANS: Reduce Operator Tree: Join Operator condition map: - Left Semi Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col1 (type: bigint) 1 _col0 (type: bigint) @@ -4048,17 +4042,11 @@ STAGE PLANS: expressions: _col1 (type: bigint) outputColumnNames: _col0 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Join Operator @@ -4127,7 +4115,7 @@ STAGE PLANS: Reduce Operator Tree: Join Operator condition map: - Left Semi Join 0 to 1 + Inner Join 0 to 1 keys: 0 _col1 (type: bigint) 1 _col0 (type: bigint) diff --git a/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out b/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out index ff801a2b29..e47a922bcd 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out @@ -2367,8 +2367,8 @@ POSTHOOK: Input: default@part 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join JOIN[34][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 8' is a cross product -Warning: Shuffle Join JOIN[45][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 4' is a cross product PREHOOK: query: explain select key, count(*) from src where value <> (select max(value) from src) group by key having count(*) > (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -2385,39 +2385,28 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 11 <- Map 10 (GROUP, 1) - Reducer 2 <- Map 1 (GROUP, 2) - Reducer 3 <- Reducer 2 (GROUP, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1), Reducer 6 (PARTITION-LEVEL SORT, 1), Reducer 9 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (GROUP, 2) - Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 1), Reducer 11 (PARTITION-LEVEL SORT, 1) - Reducer 9 <- Reducer 8 (GROUP, 2) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 6 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Reducer 2 (GROUP, 2) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1), Reducer 8 (PARTITION-LEVEL SORT, 1) + Reducer 6 <- Map 5 (GROUP, 1) + Reducer 8 <- Map 7 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s1 - filterExpr: (key = '90') (type: boolean) + alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: vectorized - Map 10 + Map 5 Map Operator Tree: TableScan alias: src @@ -2437,7 +2426,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Execution mode: vectorized - Map 5 + Map 7 Map Operator Tree: TableScan alias: s1 @@ -2462,95 +2451,89 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: vectorized - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Execution mode: vectorized - Reducer 11 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: max(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) Reducer 2 - Execution mode: vectorized Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: boolean) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 97812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col1 <> _col2) (type: boolean) + Statistics: Num rows: 500 Data size: 97812 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + Statistics: Num rows: 500 Data size: 97812 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 97812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 97812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reducer 3 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 48906 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (sq_count_check(_col0) <= 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 250 Data size: 48906 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 250 Data size: 48906 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: bigint) Reducer 4 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 - Inner Join 0 to 2 keys: 0 1 - 2 - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 31250 Data size: 6726500 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 31250 Data size: 6476500 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col3 > _col1) (type: boolean) - Statistics: Num rows: 10416 Data size: 2242023 Basic stats: COMPLETE Column stats: NONE + predicate: (_col1 > _col2) (type: boolean) + Statistics: Num rows: 10416 Data size: 2158695 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col3 (type: bigint) + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10416 Data size: 2242023 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10416 Data size: 2158695 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 10416 Data size: 2242023 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10416 Data size: 2158695 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 6 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 8 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -2574,52 +2557,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 8 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 97812 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col1 <> _col2) (type: boolean) - Statistics: Num rows: 500 Data size: 97812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 97812 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 97812 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 97812 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Reducer 9 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 48906 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 250 Data size: 48906 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 250 Data size: 48906 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -2627,8 +2564,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[34][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 8' is a cross product -Warning: Shuffle Join JOIN[45][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product +Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 4' is a cross product PREHOOK: query: select key, count(*) from src where value <> (select max(value) from src) group by key having count(*) > (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -6829,7 +6766,7 @@ POSTHOOK: query: drop table tempty_n0 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@tempty_n0 POSTHOOK: Output: default@tempty_n0 -Warning: Shuffle Join JOIN[32][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[17][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select key, count(*) from src group by key having count(*) > (select count(*) from src s1 group by 4) PREHOOK: type: QUERY @@ -6849,32 +6786,34 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 2) - Reducer 3 <- Reducer 2 (GROUP, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1), Reducer 6 (PARTITION-LEVEL SORT, 1), Reducer 8 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (GROUP, 2) - Reducer 8 <- Map 7 (GROUP, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: true (type: boolean) + aggregations: count() + keys: key (type: string) minReductionHashAggr: 0.99 mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: boolean) + key expressions: _col0 (type: string) sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Execution mode: vectorized - Map 5 + Map 4 Map Operator Tree: TableScan alias: s1 @@ -6895,92 +6834,47 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) Execution mode: vectorized - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized Reducer 2 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: boolean) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 3 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (sq_count_check(_col0) <= 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 4 + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: bigint) + Reducer 3 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 - Inner Join 0 to 2 keys: 0 1 - 2 - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 6906 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 4906 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col3 > _col1) (type: boolean) - Statistics: Num rows: 83 Data size: 2292 Basic stats: COMPLETE Column stats: NONE + predicate: (_col1 > _col2) (type: boolean) + Statistics: Num rows: 83 Data size: 1628 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col3 (type: bigint) + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 2292 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 1628 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 83 Data size: 2292 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 1628 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -7004,22 +6898,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reducer 8 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -7027,7 +6905,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[34][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[18][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 3' is a cross product PREHOOK: query: explain select key, count(*) from src group by key having count(*) > (select count(*) from src s1 where s1.key = '90' group by s1.key ) PREHOOK: type: QUERY @@ -7047,36 +6925,34 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 2) - Reducer 3 <- Reducer 2 (GROUP, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1), Reducer 6 (PARTITION-LEVEL SORT, 1), Reducer 8 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (GROUP, 2) - Reducer 8 <- Map 7 (GROUP, 2) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 5 <- Map 4 (GROUP, 2) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s1 - filterExpr: (key = '90') (type: boolean) + alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: true (type: boolean) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: boolean) - sort order: + - Map-reduce partition columns: _col0 (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + keys: key (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Execution mode: vectorized - Map 5 + Map 4 Map Operator Tree: TableScan alias: s1 @@ -7101,92 +6977,47 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Execution mode: vectorized - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: key (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Execution mode: vectorized Reducer 2 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: boolean) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 3 Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (sq_count_check(_col0) <= 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reducer 4 + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: bigint) + Reducer 3 Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 - Inner Join 0 to 2 keys: 0 1 - 2 - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 31250 Data size: 945250 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 31250 Data size: 695250 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col3 > _col1) (type: boolean) - Statistics: Num rows: 10416 Data size: 315063 Basic stats: COMPLETE Column stats: NONE + predicate: (_col1 > _col2) (type: boolean) + Statistics: Num rows: 10416 Data size: 231735 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col3 (type: bigint) + expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10416 Data size: 315063 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10416 Data size: 231735 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 10416 Data size: 315063 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 10416 Data size: 231735 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 6 + Reducer 5 Execution mode: vectorized Reduce Operator Tree: Group By Operator @@ -7210,22 +7041,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Reducer 8 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col1 is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint) Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/timestamp.q.out b/ql/src/test/results/clientpositive/timestamp.q.out index aaa9998788..f06b0da823 100644 --- a/ql/src/test/results/clientpositive/timestamp.q.out +++ b/ql/src/test/results/clientpositive/timestamp.q.out @@ -34,7 +34,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE @@ -55,7 +54,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: boolean) @@ -66,20 +64,17 @@ STAGE PLANS: expressions: TIMESTAMP'2011-01-01 01:01:01' (type: timestamp) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 5 + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator - limit: 5 + limit: -1 Processor Tree: ListSink @@ -98,7 +93,6 @@ UNION ALL SELECT CAST(TIMESTAMP '2011-01-01 01:01:01.000000000' AS TIMESTAMP) AS `$f0` FROM `default`.`src`) AS `t1` GROUP BY TRUE -LIMIT 5 STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 @@ -130,8 +124,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - TopN: 5 - TopN Hash Memory Usage: 0.1 auto parallelism: false TableScan alias: src @@ -156,8 +148,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - TopN: 5 - TopN Hash Memory Usage: 0.1 auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -223,35 +213,32 @@ STAGE PLANS: expressions: TIMESTAMP'2011-01-01 01:01:01' (type: timestamp) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 5 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0 - columns.types timestamp - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types timestamp + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Fetch Operator - limit: 5 + limit: -1 Processor Tree: ListSink @@ -300,7 +287,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE @@ -321,7 +307,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: boolean) Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: boolean) @@ -332,20 +317,17 @@ STAGE PLANS: expressions: TIMESTAMP'2011-01-01 01:01:01.123' (type: timestamp) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 5 + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator - limit: 5 + limit: -1 Processor Tree: ListSink diff --git a/ql/src/test/results/clientpositive/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/vector_outer_join3.q.out index 0640aeec75..43d9ddf3c3 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join3.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join3.q.out @@ -248,7 +248,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 10.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cint\",\n \"cstring1\"\n ],\n \"exprs\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"cd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n \"rowCount\": 10.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cint\"\n ],\n \"exprs\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ],\n \"rowCount\": 10.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 25.0\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"hd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 6.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ]\n },\n \"rowCount\": 10.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cstring1\"\n ],\n \"exprs\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ],\n \"rowCount\": 10.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 31.25\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate\",\n \"group\": [],\n \"aggs\": [\n {\n \"agg\": \"count\",\n \"type\": {\n \"type\": \"BIGINT\",\n \"nullable\": true\n },\n \"distinct\": false,\n \"operands\": []\n }\n ],\n \"rowCount\": 1.0\n }\n ]\n}","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cstring1"],"database:":"default","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 6]"},"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col1":"0:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col1"],"Statistics:":"Num rows: 45 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 101 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 6]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 10.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cint\",\n \"cstring1\"\n ],\n \"exprs\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"cd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n \"rowCount\": 10.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cint\"\n ],\n \"exprs\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ],\n \"rowCount\": 10.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 25.0\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"hd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 6.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ]\n },\n \"rowCount\": 10.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cstring1\"\n ],\n \"exprs\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ],\n \"rowCount\": 10.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 31.25\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate\",\n \"group\": [],\n \"aggs\": [\n {\n \"agg\": {\n \"name\": \"count\",\n \"kind\": \"COUNT\",\n \"syntax\": \"FUNCTION_STAR\"\n },\n \"type\": {\n \"type\": \"BIGINT\",\n \"nullable\": true\n },\n \"distinct\": false,\n \"operands\": [],\n \"name\": null\n }\n ],\n \"rowCount\": 1.0\n }\n ]\n}","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cstring1"],"database:":"default","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 6]"},"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col1":"0:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col1"],"Statistics:":"Num rows: 45 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 101 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 6]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd @@ -292,7 +292,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 14.75,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cstring1\",\n \"cstring2\"\n ],\n \"exprs\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n },\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"cd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 8.75,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ]\n },\n \"rowCount\": 15.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cstring2\"\n ],\n \"exprs\": [\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ],\n \"rowCount\": 15.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"hd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 6.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ]\n },\n \"rowCount\": 10.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cstring1\"\n ],\n \"exprs\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ],\n \"rowCount\": 10.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 25.0\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate\",\n \"group\": [],\n \"aggs\": [\n {\n \"agg\": \"count\",\n \"type\": {\n \"type\": \"BIGINT\",\n \"nullable\": true\n },\n \"distinct\": false,\n \"operands\": []\n }\n ],\n \"rowCount\": 1.0\n }\n ]\n}","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cstring2"],"database:":"default","filterExpr:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 1488 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cstring2 (type: string)","columnExprMap:":{"_col0":"cstring2"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cstring1","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[6, 7]"},"Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 7:string"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 23 Data size: 1260 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 51 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 14.75,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cstring1\",\n \"cstring2\"\n ],\n \"exprs\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n },\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"cd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 8.75,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ]\n },\n \"rowCount\": 15.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cstring2\"\n ],\n \"exprs\": [\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ],\n \"rowCount\": 15.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"hd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 6.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ]\n },\n \"rowCount\": 10.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cstring1\"\n ],\n \"exprs\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ],\n \"rowCount\": 10.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 25.0\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate\",\n \"group\": [],\n \"aggs\": [\n {\n \"agg\": {\n \"name\": \"count\",\n \"kind\": \"COUNT\",\n \"syntax\": \"FUNCTION_STAR\"\n },\n \"type\": {\n \"type\": \"BIGINT\",\n \"nullable\": true\n },\n \"distinct\": false,\n \"operands\": [],\n \"name\": null\n }\n ],\n \"rowCount\": 1.0\n }\n ]\n}","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cstring2"],"database:":"default","filterExpr:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 1488 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cstring2 (type: string)","columnExprMap:":{"_col0":"cstring2"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cstring1","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[6, 7]"},"Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 7:string"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 23 Data size: 1260 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 51 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd @@ -336,7 +336,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 26.75,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cint\",\n \"cbigint\",\n \"cstring1\",\n \"cstring2\"\n ],\n \"exprs\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 6,\n \"name\": \"$6\"\n },\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"cd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 16.75,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"AND\",\n \"operands\": [\n {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n }\n ]\n },\n {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ]\n }\n ]\n },\n \"rowCount\": 11.25\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cbigint\",\n \"cstring2\"\n ],\n \"exprs\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ],\n \"rowCount\": 11.25\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"AND\",\n \"operands\": [\n {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 5,\n \"name\": \"$5\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n }\n ]\n },\n {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 4,\n \"name\": \"$4\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"hd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 10.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"AND\",\n \"operands\": [\n {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ]\n }\n ]\n },\n \"rowCount\": 5.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cint\",\n \"cstring1\"\n ],\n \"exprs\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ],\n \"rowCount\": 5.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"AND\",\n \"operands\": [\n {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 7,\n \"name\": \"$7\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate\",\n \"group\": [],\n \"aggs\": [\n {\n \"agg\": \"count\",\n \"type\": {\n \"type\": \"BIGINT\",\n \"nullable\": true\n },\n \"distinct\": false,\n \"operands\": []\n }\n ],\n \"rowCount\": 1.0\n }\n ]\n}","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cbigint`, `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cbigint`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cbigint` IS NOT NULL AND `cstring2` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2` AND `t`.`cbigint` = `t1`.`cbigint`\nLEFT JOIN (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cint` IS NOT NULL AND `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1` AND `t`.`cint` = `t3`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cbigint","cstring2"],"database:":"default","filterExpr:":"(cbigint is not null and cstring2 is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"(cbigint is not null and cstring2 is not null) (type: boolean)","Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cbigint (type: bigint), cstring2 (type: string)","columnExprMap:":{"_col0":"cbigint","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cint","cstring1"],"database:":"default","filterExpr:":"(cint is not null and cstring1 is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"(cint is not null and cstring1 is not null) (type: boolean)","Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cbigint","cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cbigint","_col2":"cstring1","_col3":"cstring2"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 3, 6, 7]"},"Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 3:bigint","col 7:string"],"bigTableValueExpressions:":["col 2:int","col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 26 Data size: 1598 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:int","col 1:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 42 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 3, 6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 26.75,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cint\",\n \"cbigint\",\n \"cstring1\",\n \"cstring2\"\n ],\n \"exprs\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 6,\n \"name\": \"$6\"\n },\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"cd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 16.75,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"AND\",\n \"kind\": \"AND\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n }\n ]\n },\n {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ]\n }\n ]\n },\n \"rowCount\": 11.25\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cbigint\",\n \"cstring2\"\n ],\n \"exprs\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 7,\n \"name\": \"$7\"\n }\n ],\n \"rowCount\": 11.25\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"AND\",\n \"kind\": \"AND\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 5,\n \"name\": \"$5\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n }\n ]\n },\n {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 4,\n \"name\": \"$4\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_a_n1\"\n ],\n \"table:alias\": \"hd\",\n \"inputs\": [],\n \"rowCount\": 20.0,\n \"avgRowSize\": 10.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cint\",\n \"ndv\": 8,\n \"minValue\": -738306196,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 8\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 2,\n \"minValue\": -64,\n \"maxValue\": -51\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 6,\n \"minValue\": -15920,\n \"maxValue\": -6907\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 15,\n \"minValue\": -1970551565,\n \"maxValue\": 1086455747\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 2,\n \"minValue\": -64.0,\n \"maxValue\": -51.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 6,\n \"minValue\": -15920.0,\n \"maxValue\": -6907.0\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 15\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"AND\",\n \"kind\": \"AND\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ]\n }\n ]\n },\n \"rowCount\": 5.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cint\",\n \"cstring1\"\n ],\n \"exprs\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 6,\n \"name\": \"$6\"\n }\n ],\n \"rowCount\": 5.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"AND\",\n \"kind\": \"AND\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 7,\n \"name\": \"$7\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 6,\n \"name\": \"$6\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate\",\n \"group\": [],\n \"aggs\": [\n {\n \"agg\": {\n \"name\": \"count\",\n \"kind\": \"COUNT\",\n \"syntax\": \"FUNCTION_STAR\"\n },\n \"type\": {\n \"type\": \"BIGINT\",\n \"nullable\": true\n },\n \"distinct\": false,\n \"operands\": [],\n \"name\": null\n }\n ],\n \"rowCount\": 1.0\n }\n ]\n}","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cbigint`, `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cbigint`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cbigint` IS NOT NULL AND `cstring2` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2` AND `t`.`cbigint` = `t1`.`cbigint`\nLEFT JOIN (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cint` IS NOT NULL AND `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1` AND `t`.`cint` = `t3`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cbigint","cstring2"],"database:":"default","filterExpr:":"(cbigint is not null and cstring2 is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"(cbigint is not null and cstring2 is not null) (type: boolean)","Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cbigint (type: bigint), cstring2 (type: string)","columnExprMap:":{"_col0":"cbigint","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cint","cstring1"],"database:":"default","filterExpr:":"(cint is not null and cstring1 is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"(cint is not null and cstring1 is not null) (type: boolean)","Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cbigint","cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cbigint","_col2":"cstring1","_col3":"cstring2"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 3, 6, 7]"},"Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 3:bigint","col 7:string"],"bigTableValueExpressions:":["col 2:int","col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 26 Data size: 1598 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:int","col 1:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 42 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 3, 6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd diff --git a/ql/src/test/results/clientpositive/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/vector_outer_join4.q.out index d4bf60a755..6686575727 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join4.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join4.q.out @@ -262,7 +262,7 @@ left outer join small_alltypesorc_b cd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 139.86666666666667,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\",\n \"csmallint\",\n \"cint\",\n \"cbigint\",\n \"cfloat\",\n \"cdouble\",\n \"cstring1\",\n \"cstring2\",\n \"ctimestamp1\",\n \"ctimestamp2\",\n \"cboolean1\",\n \"cboolean2\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 4,\n \"name\": \"$4\"\n },\n {\n \"input\": 5,\n \"name\": \"$5\"\n },\n {\n \"input\": 6,\n \"name\": \"$6\"\n },\n {\n \"input\": 7,\n \"name\": \"$7\"\n },\n {\n \"input\": 8,\n \"name\": \"$8\"\n },\n {\n \"input\": 9,\n \"name\": \"$9\"\n },\n {\n \"input\": 10,\n \"name\": \"$10\"\n },\n {\n \"input\": 11,\n \"name\": \"$11\"\n }\n ],\n \"rowCount\": 30.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"cd\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 139.86666666666667,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n \"rowCount\": 20.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\",\n \"csmallint\",\n \"cint\",\n \"cbigint\",\n \"cfloat\",\n \"cdouble\",\n \"cstring1\",\n \"cstring2\",\n \"ctimestamp1\",\n \"ctimestamp2\",\n \"cboolean1\",\n \"cboolean2\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 4,\n \"name\": \"$4\"\n },\n {\n \"input\": 5,\n \"name\": \"$5\"\n },\n {\n \"input\": 6,\n \"name\": \"$6\"\n },\n {\n \"input\": 7,\n \"name\": \"$7\"\n },\n {\n \"input\": 8,\n \"name\": \"$8\"\n },\n {\n \"input\": 9,\n \"name\": \"$9\"\n },\n {\n \"input\": 10,\n \"name\": \"$10\"\n },\n {\n \"input\": 11,\n \"name\": \"$11\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 14,\n \"name\": \"$14\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 46.15384615384615\n }\n ]\n}","optimizedSQL":"SELECT *\nFROM (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_10","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"OperatorId:":"HASHTABLESINK_12"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]"},"Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_14","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col1":"0:_col1","_col10":"0:_col10","_col11":"0:_col11","_col12":"1:_col0","_col13":"1:_col1","_col14":"1:_col2","_col15":"1:_col3","_col16":"1:_col4","_col17":"1:_col5","_col18":"1:_col6","_col19":"1:_col7","_col2":"0:_col2","_col20":"1:_col8","_col21":"1:_col9","_col22":"1:_col10","_col23":"1:_col11","_col3":"0:_col3","_col4":"0:_col4","_col5":"0:_col5","_col6":"0:_col6","_col7":"0:_col7","_col8":"0:_col8","_col9":"0:_col9"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint","col 1:smallint","col 2:int","col 3:bigint","col 4:float","col 5:double","col 6:string","col 7:string","col 8:timestamp","col 9:timestamp","col 10:boolean","col 11:boolean"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"],"Statistics:":"Num rows: 76 Data size: 34428 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_15","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 76 Data size: 34428 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint, bigint, bigint, double, double, string, string, timestamp, timestamp, bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 139.86666666666667,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\",\n \"csmallint\",\n \"cint\",\n \"cbigint\",\n \"cfloat\",\n \"cdouble\",\n \"cstring1\",\n \"cstring2\",\n \"ctimestamp1\",\n \"ctimestamp2\",\n \"cboolean1\",\n \"cboolean2\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 4,\n \"name\": \"$4\"\n },\n {\n \"input\": 5,\n \"name\": \"$5\"\n },\n {\n \"input\": 6,\n \"name\": \"$6\"\n },\n {\n \"input\": 7,\n \"name\": \"$7\"\n },\n {\n \"input\": 8,\n \"name\": \"$8\"\n },\n {\n \"input\": 9,\n \"name\": \"$9\"\n },\n {\n \"input\": 10,\n \"name\": \"$10\"\n },\n {\n \"input\": 11,\n \"name\": \"$11\"\n }\n ],\n \"rowCount\": 30.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"cd\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 139.86666666666667,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n \"rowCount\": 20.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\",\n \"csmallint\",\n \"cint\",\n \"cbigint\",\n \"cfloat\",\n \"cdouble\",\n \"cstring1\",\n \"cstring2\",\n \"ctimestamp1\",\n \"ctimestamp2\",\n \"cboolean1\",\n \"cboolean2\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 4,\n \"name\": \"$4\"\n },\n {\n \"input\": 5,\n \"name\": \"$5\"\n },\n {\n \"input\": 6,\n \"name\": \"$6\"\n },\n {\n \"input\": 7,\n \"name\": \"$7\"\n },\n {\n \"input\": 8,\n \"name\": \"$8\"\n },\n {\n \"input\": 9,\n \"name\": \"$9\"\n },\n {\n \"input\": 10,\n \"name\": \"$10\"\n },\n {\n \"input\": 11,\n \"name\": \"$11\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 14,\n \"name\": \"$14\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 46.15384615384615\n }\n ]\n}","optimizedSQL":"SELECT *\nFROM (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_10","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"OperatorId:":"HASHTABLESINK_12"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]"},"Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_14","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col1":"0:_col1","_col10":"0:_col10","_col11":"0:_col11","_col12":"1:_col0","_col13":"1:_col1","_col14":"1:_col2","_col15":"1:_col3","_col16":"1:_col4","_col17":"1:_col5","_col18":"1:_col6","_col19":"1:_col7","_col2":"0:_col2","_col20":"1:_col8","_col21":"1:_col9","_col22":"1:_col10","_col23":"1:_col11","_col3":"0:_col3","_col4":"0:_col4","_col5":"0:_col5","_col6":"0:_col6","_col7":"0:_col7","_col8":"0:_col8","_col9":"0:_col9"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint","col 1:smallint","col 2:int","col 3:bigint","col 4:float","col 5:double","col 6:string","col 7:string","col 8:timestamp","col 9:timestamp","col 10:boolean","col 11:boolean"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"],"Statistics:":"Num rows: 76 Data size: 34428 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_15","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 76 Data size: 34428 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint, bigint, bigint, double, double, string, string, timestamp, timestamp, bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} PREHOOK: query: select * from small_alltypesorc_b c left outer join small_alltypesorc_b cd @@ -347,7 +347,7 @@ left outer join small_alltypesorc_b hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 30.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"hd\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"rowCount\": 20.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 600.0\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 600.0\n }\n ]\n}","optimizedSQL":"SELECT `t`.`ctinyint`\nFROM (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_10","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_12"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_14","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_15","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 30.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"hd\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"rowCount\": 20.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 600.0\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 600.0\n }\n ]\n}","optimizedSQL":"SELECT `t`.`ctinyint`\nFROM (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_10","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_12"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_14","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_15","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} PREHOOK: query: select c.ctinyint from small_alltypesorc_b c left outer join small_alltypesorc_b hd @@ -794,7 +794,7 @@ left outer join small_alltypesorc_b hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\",\n \"cint\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ],\n \"rowCount\": 30.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"cd\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n \"rowCount\": 20.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cint\"\n ],\n \"exprs\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 46.15384615384615\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"hd\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"rowCount\": 20.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 923.0769230769231\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate\",\n \"group\": [],\n \"aggs\": [\n {\n \"agg\": \"count\",\n \"type\": {\n \"type\": \"BIGINT\",\n \"nullable\": true\n },\n \"distinct\": false,\n \"operands\": []\n }\n ],\n \"rowCount\": 1.0\n }\n ]\n}","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `cint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t3` ON `t`.`ctinyint` = `t3`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","cint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), cint (type: int)","columnExprMap:":{"_col0":"ctinyint","_col1":"cint"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 2]"},"Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 58 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1160 Data size: 9280 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 2]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"c\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\",\n \"cint\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ],\n \"rowCount\": 30.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"cd\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ]\n },\n \"rowCount\": 20.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"cint\"\n ],\n \"exprs\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 46.15384615384615\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"small_alltypesorc_b\"\n ],\n \"table:alias\": \"hd\",\n \"inputs\": [],\n \"rowCount\": 30.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"TINYINT\",\n \"nullable\": true,\n \"name\": \"ctinyint\"\n },\n {\n \"type\": \"SMALLINT\",\n \"nullable\": true,\n \"name\": \"csmallint\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"cint\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"cbigint\"\n },\n {\n \"type\": \"FLOAT\",\n \"nullable\": true,\n \"name\": \"cfloat\"\n },\n {\n \"type\": \"DOUBLE\",\n \"nullable\": true,\n \"name\": \"cdouble\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring1\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"cstring2\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp1\"\n },\n {\n \"type\": \"TIMESTAMP\",\n \"nullable\": true,\n \"precision\": 9,\n \"name\": \"ctimestamp2\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean1\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"cboolean2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"ctinyint\",\n \"ndv\": 1,\n \"minValue\": -64,\n \"maxValue\": -64\n },\n {\n \"name\": \"csmallint\",\n \"ndv\": 21,\n \"minValue\": -16379,\n \"maxValue\": -200\n },\n {\n \"name\": \"cint\",\n \"ndv\": 13,\n \"minValue\": -894716315,\n \"maxValue\": 626923679\n },\n {\n \"name\": \"cbigint\",\n \"ndv\": 13,\n \"minValue\": -1887561756,\n \"maxValue\": 2118653994\n },\n {\n \"name\": \"cfloat\",\n \"ndv\": 1,\n \"minValue\": -64.0,\n \"maxValue\": -64.0\n },\n {\n \"name\": \"cdouble\",\n \"ndv\": 21,\n \"minValue\": -16379.0,\n \"maxValue\": -200.0\n },\n {\n \"name\": \"cstring1\",\n \"ndv\": 13\n },\n {\n \"name\": \"cstring2\",\n \"ndv\": 13\n },\n {\n \"name\": \"ctimestamp1\",\n \"ndv\": 0\n },\n {\n \"name\": \"ctimestamp2\",\n \"ndv\": 0\n },\n {\n \"name\": \"cboolean1\",\n \"ndv\": 2\n },\n {\n \"name\": \"cboolean2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"rowCount\": 20.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"ctinyint\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 20.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 923.0769230769231\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate\",\n \"group\": [],\n \"aggs\": [\n {\n \"agg\": {\n \"name\": \"count\",\n \"kind\": \"COUNT\",\n \"syntax\": \"FUNCTION_STAR\"\n },\n \"type\": {\n \"type\": \"BIGINT\",\n \"nullable\": true\n },\n \"distinct\": false,\n \"operands\": [],\n \"name\": null\n }\n ],\n \"rowCount\": 1.0\n }\n ]\n}","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `cint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t3` ON `t`.`ctinyint` = `t3`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","cint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), cint (type: int)","columnExprMap:":{"_col0":"ctinyint","_col1":"cint"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 2]"},"Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 58 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1160 Data size: 9280 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 2]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.ctinyint from small_alltypesorc_b c left outer join small_alltypesorc_b cd diff --git a/ql/src/test/results/clientpositive/vector_outer_join6.q.out b/ql/src/test/results/clientpositive/vector_outer_join6.q.out index 174773d5ec..0725e774de 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join6.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join6.q.out @@ -138,7 +138,7 @@ POSTHOOK: Input: default@tjoin1_n0 POSTHOOK: Input: default@tjoin2_n0 POSTHOOK: Input: default@tjoin3 #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin1_n0\"\n ],\n \"table:alias\": \"tjoin1_n0\",\n \"inputs\": [],\n \"rowCount\": 3.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"rnum\",\n \"ndv\": 3,\n \"minValue\": 0,\n \"maxValue\": 2\n },\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 20\n },\n {\n \"name\": \"c2\",\n \"ndv\": 3,\n \"minValue\": 15,\n \"maxValue\": 50\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"rnum\",\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 3.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin2_n0\"\n ],\n \"table:alias\": \"tjoin2_n0\",\n \"inputs\": [],\n \"rowCount\": 4.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"CHAR\",\n \"nullable\": true,\n \"precision\": 2,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"rnum\",\n \"ndv\": 4,\n \"minValue\": 0,\n \"maxValue\": 3\n },\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 15\n },\n {\n \"name\": \"c2\",\n \"ndv\": 4\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"rowCount\": 3.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"rnum\",\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 3.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 4.5\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin3\"\n ],\n \"table:alias\": \"tjoin3\",\n \"inputs\": [],\n \"rowCount\": 2.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"CHAR\",\n \"nullable\": true,\n \"precision\": 2,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"rnum\",\n \"ndv\": 2,\n \"minValue\": 0,\n \"maxValue\": 1\n },\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 15\n },\n {\n \"name\": \"c2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"rowCount\": 2.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"rnum\",\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 2.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 5,\n \"name\": \"$5\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 4.5\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"tj1rnum\",\n \"tj2rnum\",\n \"rnumt3\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 4,\n \"name\": \"$4\"\n }\n ],\n \"rowCount\": 4.5\n }\n ]\n}","optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `tj2rnum`, `t3`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_16","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_22"}}}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2","_col4":"1:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col4"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_28","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int), _col4 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2","_col2":"_col4"},"outputColumnNames:":["_col0","_col1","_col2"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2]"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_29","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_30"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_31"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin1_n0\"\n ],\n \"table:alias\": \"tjoin1_n0\",\n \"inputs\": [],\n \"rowCount\": 3.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"rnum\",\n \"ndv\": 3,\n \"minValue\": 0,\n \"maxValue\": 2\n },\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 20\n },\n {\n \"name\": \"c2\",\n \"ndv\": 3,\n \"minValue\": 15,\n \"maxValue\": 50\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"rnum\",\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 3.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin2_n0\"\n ],\n \"table:alias\": \"tjoin2_n0\",\n \"inputs\": [],\n \"rowCount\": 4.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"CHAR\",\n \"nullable\": true,\n \"precision\": 2,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"rnum\",\n \"ndv\": 4,\n \"minValue\": 0,\n \"maxValue\": 3\n },\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 15\n },\n {\n \"name\": \"c2\",\n \"ndv\": 4\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"rowCount\": 3.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"rnum\",\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 3.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 4.5\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin3\"\n ],\n \"table:alias\": \"tjoin3\",\n \"inputs\": [],\n \"rowCount\": 2.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"CHAR\",\n \"nullable\": true,\n \"precision\": 2,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"rnum\",\n \"ndv\": 2,\n \"minValue\": 0,\n \"maxValue\": 1\n },\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 15\n },\n {\n \"name\": \"c2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"rowCount\": 2.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"rnum\",\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 2.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 5,\n \"name\": \"$5\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 4.5\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"tj1rnum\",\n \"tj2rnum\",\n \"rnumt3\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n },\n {\n \"input\": 4,\n \"name\": \"$4\"\n }\n ],\n \"rowCount\": 4.5\n }\n ]\n}","optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `tj2rnum`, `t3`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_16","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_22"}}}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2","_col4":"1:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col4"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_28","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int), _col4 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2","_col2":"_col4"},"outputColumnNames:":["_col0","_col1","_col2"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2]"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_29","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_30"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_31"}}}}}} PREHOOK: query: select tj1rnum, tj2rnum, tjoin3.rnum as rnumt3 from (select tjoin1_n0.rnum tj1rnum, tjoin2_n0.rnum tj2rnum, tjoin2_n0.c1 tj2c1 from tjoin1_n0 left outer join tjoin2_n0 on tjoin1_n0.c1 = tjoin2_n0.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1 PREHOOK: type: QUERY @@ -173,7 +173,7 @@ POSTHOOK: Input: default@tjoin1_n0 POSTHOOK: Input: default@tjoin2_n0 POSTHOOK: Input: default@tjoin3 #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin1_n0\"\n ],\n \"table:alias\": \"tjoin1_n0\",\n \"inputs\": [],\n \"rowCount\": 3.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"rnum\",\n \"ndv\": 3,\n \"minValue\": 0,\n \"maxValue\": 2\n },\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 20\n },\n {\n \"name\": \"c2\",\n \"ndv\": 3,\n \"minValue\": 15,\n \"maxValue\": 50\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"rnum\",\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 3.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin2_n0\"\n ],\n \"table:alias\": \"tjoin2_n0\",\n \"inputs\": [],\n \"rowCount\": 4.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"CHAR\",\n \"nullable\": true,\n \"precision\": 2,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"rnum\",\n \"ndv\": 4,\n \"minValue\": 0,\n \"maxValue\": 3\n },\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 15\n },\n {\n \"name\": \"c2\",\n \"ndv\": 4\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"rowCount\": 3.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"rnum\",\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 3.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 4.5\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin3\"\n ],\n \"table:alias\": \"tjoin3\",\n \"inputs\": [],\n \"rowCount\": 2.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"CHAR\",\n \"nullable\": true,\n \"precision\": 2,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 15\n },\n {\n \"name\": \"rnum\",\n \"ndv\": 2,\n \"minValue\": 0,\n \"maxValue\": 1\n },\n {\n \"name\": \"c2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": \"IS NOT NULL\",\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"rowCount\": 2.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 2.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": \"=\",\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 4,\n \"name\": \"$4\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 4.5\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"tj1rnum\",\n \"rnumt3\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ],\n \"rowCount\": 4.5\n }\n ]\n}","optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_16","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"c1 (type: int)","columnExprMap:":{"_col0":"c1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_22"}}}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_28","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_29","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_30"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_31"}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin1_n0\"\n ],\n \"table:alias\": \"tjoin1_n0\",\n \"inputs\": [],\n \"rowCount\": 3.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"rnum\",\n \"ndv\": 3,\n \"minValue\": 0,\n \"maxValue\": 2\n },\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 20\n },\n {\n \"name\": \"c2\",\n \"ndv\": 3,\n \"minValue\": 15,\n \"maxValue\": 50\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"rnum\",\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 3.0\n },\n {\n \"id\": \"2\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin2_n0\"\n ],\n \"table:alias\": \"tjoin2_n0\",\n \"inputs\": [],\n \"rowCount\": 4.0,\n \"avgRowSize\": 8.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"CHAR\",\n \"nullable\": true,\n \"precision\": 2,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"rnum\",\n \"ndv\": 4,\n \"minValue\": 0,\n \"maxValue\": 3\n },\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 15\n },\n {\n \"name\": \"c2\",\n \"ndv\": 4\n }\n ]\n },\n {\n \"id\": \"3\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"rowCount\": 3.0\n },\n {\n \"id\": \"4\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"rnum\",\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 3.0\n },\n {\n \"id\": \"5\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 3,\n \"name\": \"$3\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"1\",\n \"4\"\n ],\n \"rowCount\": 4.5\n },\n {\n \"id\": \"6\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"tjoin3\"\n ],\n \"table:alias\": \"tjoin3\",\n \"inputs\": [],\n \"rowCount\": 2.0,\n \"avgRowSize\": 4.0,\n \"rowType\": [\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"rnum\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"c1\"\n },\n {\n \"type\": \"CHAR\",\n \"nullable\": true,\n \"precision\": 2,\n \"name\": \"c2\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n }\n ],\n \"colStats\": [\n {\n \"name\": \"c1\",\n \"ndv\": 2,\n \"minValue\": 10,\n \"maxValue\": 15\n },\n {\n \"name\": \"rnum\",\n \"ndv\": 2,\n \"minValue\": 0,\n \"maxValue\": 1\n },\n {\n \"name\": \"c2\",\n \"ndv\": 2\n }\n ]\n },\n {\n \"id\": \"7\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter\",\n \"condition\": {\n \"op\": {\n \"name\": \"IS NOT NULL\",\n \"kind\": \"IS_NOT_NULL\",\n \"syntax\": \"POSTFIX\"\n },\n \"operands\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ]\n },\n \"rowCount\": 2.0\n },\n {\n \"id\": \"8\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"c1\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n }\n ],\n \"rowCount\": 2.0\n },\n {\n \"id\": \"9\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin\",\n \"condition\": {\n \"op\": {\n \"name\": \"=\",\n \"kind\": \"EQUALS\",\n \"syntax\": \"BINARY\"\n },\n \"operands\": [\n {\n \"input\": 3,\n \"name\": \"$3\"\n },\n {\n \"input\": 4,\n \"name\": \"$4\"\n }\n ]\n },\n \"joinType\": \"left\",\n \"algorithm\": \"none\",\n \"cost\": \"not available\",\n \"inputs\": [\n \"5\",\n \"8\"\n ],\n \"rowCount\": 4.5\n },\n {\n \"id\": \"10\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"tj1rnum\",\n \"rnumt3\"\n ],\n \"exprs\": [\n {\n \"input\": 0,\n \"name\": \"$0\"\n },\n {\n \"input\": 2,\n \"name\": \"$2\"\n }\n ],\n \"rowCount\": 4.5\n }\n ]\n}","optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_16","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"c1 (type: int)","columnExprMap:":{"_col0":"c1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_22"}}}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_28","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_29","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_30"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_31"}}}}}} PREHOOK: query: select tj1rnum, tj2rnum as rnumt3 from (select tjoin1_n0.rnum tj1rnum, tjoin2_n0.rnum tj2rnum, tjoin2_n0.c1 tj2c1 from tjoin1_n0 left outer join tjoin2_n0 on tjoin1_n0.c1 = tjoin2_n0.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/vectorization_limit.q.out b/ql/src/test/results/clientpositive/vectorization_limit.q.out index d91fa77341..5a63cc7a8b 100644 --- a/ql/src/test/results/clientpositive/vectorization_limit.q.out +++ b/ql/src/test/results/clientpositive/vectorization_limit.q.out @@ -701,12 +701,75 @@ PLAN VECTORIZATION: enabledConditionsMet: [hive.vectorized.execution.enabled IS true] STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 5] + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: double) + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 12 + includeColumns: [0, 5] + dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 12288 Data size: 110096 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator - limit: 0 + limit: -1 Processor Tree: ListSink