diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java index 50fbb78a94..268284a6da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java @@ -21,13 +21,19 @@ import java.util.ArrayList; import java.util.List; +import com.google.common.collect.ImmutableList; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; @@ -347,4 +353,111 @@ public String get(int index) { }, true, relBuilder); } + public static RexNode splitCorrelatedFilterCondition( + Filter filter, + List joinKeys, + List correlatedJoinKeys, + boolean extractCorrelatedFieldAccess) { + final List nonEquiList = new ArrayList<>(); + + splitCorrelatedFilterCondition( + filter, + filter.getCondition(), + joinKeys, + correlatedJoinKeys, + nonEquiList, + extractCorrelatedFieldAccess); + + // Convert the remainders into a list that are AND'ed together. + return RexUtil.composeConjunction( + filter.getCluster().getRexBuilder(), nonEquiList, true); + } + + private static void splitCorrelatedFilterCondition( + Filter filter, + RexNode condition, + List joinKeys, + List correlatedJoinKeys, + List nonEquiList, + boolean extractCorrelatedFieldAccess) { + if (condition instanceof RexCall) { + RexCall call = (RexCall) condition; + if (call.getOperator().getKind() == SqlKind.AND) { + for (RexNode operand : call.getOperands()) { + splitCorrelatedFilterCondition( + filter, + operand, + joinKeys, + correlatedJoinKeys, + nonEquiList, + extractCorrelatedFieldAccess); + } + return; + } + + if (call.getOperator().getKind() == SqlKind.EQUALS) { + final List operands = call.getOperands(); + RexNode op0 = operands.get(0); + RexNode op1 = operands.get(1); + + if (extractCorrelatedFieldAccess) { + if (!RexUtil.containsFieldAccess(op0) + && (op1 instanceof RexFieldAccess)) { + joinKeys.add(op0); + correlatedJoinKeys.add(op1); + return; + } else if ( + (op0 instanceof RexFieldAccess) + && !RexUtil.containsFieldAccess(op1)) { + correlatedJoinKeys.add(op0); + joinKeys.add(op1); + return; + } + } else { + if (!(RexUtil.containsInputRef(op0)) + && (op1 instanceof RexInputRef)) { + correlatedJoinKeys.add(op0); + joinKeys.add(op1); + return; + } else if ( + (op0 instanceof RexInputRef) + && !(RexUtil.containsInputRef(op1))) { + joinKeys.add(op0); + correlatedJoinKeys.add(op1); + return; + } + } + } + } + + // The operator is not of RexCall type + // So we fail. Fall through. + // Add this condition to the list of non-equi-join conditions. + nonEquiList.add(condition); + } + + /** + * Creates a LogicalAggregate that removes all duplicates from the result of + * an underlying relational expression. + * + * @param rel underlying rel + * @return rel implementing SingleValueAgg + */ + public static RelNode createSingleValueAggRel( + RelOptCluster cluster, + RelNode rel, + RelFactories.AggregateFactory aggregateFactory) { + // assert (rel.getRowType().getFieldCount() == 1); + final int aggCallCnt = rel.getRowType().getFieldCount(); + final List aggCalls = new ArrayList<>(); + + for (int i = 0; i < aggCallCnt; i++) { + aggCalls.add( + AggregateCall.create( + SqlStdOperatorTable.SINGLE_VALUE, false, false, + ImmutableList.of(i), -1, 0, rel, null, null)); + } + + return aggregateFactory.createAggregate(rel, false, ImmutableBitSet.of(), null, aggCalls); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java index 23ff518ada..5dcbff6659 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveVolcanoPlanner.java @@ -97,7 +97,10 @@ public RelOptCost getCost(RelNode rel, RelMetadataQuery mq) { if (rel instanceof RelSubset) { // Get cost of the subset, best rel may have been chosen or not RelSubset subset = (RelSubset) rel; - return getCost(Util.first(subset.getBest(), subset.getOriginal()), mq); + if (subset.getBest() != null) { + return getCost(subset.getBest(), mq); + } + return costFactory.makeInfiniteCost(); } if (rel.getTraitSet().getTrait(ConventionTraitDef.INSTANCE) == Convention.NONE) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java index 24e22a0bf3..7e3d495933 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java @@ -53,22 +53,14 @@ import org.apache.calcite.rel.core.Correlate; import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.Values; -import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalCorrelate; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.calcite.rel.logical.LogicalIntersect; -import org.apache.calcite.rel.logical.LogicalJoin; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rel.logical.LogicalUnion; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rel.rules.FilterJoinRule; -import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; @@ -104,6 +96,8 @@ import org.apache.calcite.util.Stacks; import org.apache.calcite.util.Util; import org.apache.calcite.util.mapping.Mappings; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttleImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; @@ -137,7 +131,7 @@ * NOTE: this whole logic is replicated from Calcite's RelDecorrelator * and is exteneded to make it suitable for HIVE * We should get rid of this and replace it with Calcite's RelDecorrelator - * once that works with Join, Project etc instead of LogicalJoin, LogicalProject. + * once that works with Join, Project etc instead of Join, Project. * At this point this has differed from Calcite's version significantly so cannot * get rid of this. * @@ -198,8 +192,7 @@ private HiveRelDecorrelator( this.cm = cm; this.rexBuilder = cluster.getRexBuilder(); this.context = context; - relBuilder = RelFactories.LOGICAL_BUILDER.create(cluster, null); - + relBuilder = HiveRelFactories.HIVE_BUILDER.create(cluster, null); } //~ Methods ---------------------------------------------------------------- @@ -245,8 +238,8 @@ private RelNode decorrelate(RelNode root) { HepProgram program = HepProgram.builder() .addRuleInstance(new AdjustProjectForCountAggregateRule(false)) .addRuleInstance(new AdjustProjectForCountAggregateRule(true)) - .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN) - .addRuleInstance(FilterProjectTransposeRule.INSTANCE) + .addRuleInstance(HiveFilterJoinRule.FILTER_ON_JOIN) + .addRuleInstance(HiveFilterProjectTransposeRule.INSTANCE) // FilterCorrelateRule rule mistakenly pushes a FILTER, consiting of correlated vars, // on top of LogicalCorrelate to within left input for scalar corr queries // which causes exception during decorrelation. This has been disabled for now. @@ -265,8 +258,8 @@ private RelNode decorrelate(RelNode root) { if (frame != null) { // has been rewritten; apply rules post-decorrelation final HepProgram program2 = HepProgram.builder() - .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN) - .addRuleInstance(FilterJoinRule.JOIN) + .addRuleInstance(HiveFilterJoinRule.FILTER_ON_JOIN) + .addRuleInstance(HiveFilterJoinRule.JOIN) .build(); final HepPlanner planner2 = createPlanner(program2); @@ -504,11 +497,11 @@ public Frame decorrelateRel(Values rel) { } /** - * Rewrites a {@link LogicalAggregate}. + * Rewrites a {@link Aggregate}. * * @param rel Aggregate to rewrite */ - public Frame decorrelateRel(LogicalAggregate rel) throws SemanticException{ + public Frame decorrelateRel(Aggregate rel) throws SemanticException{ if (rel.getGroupType() != Aggregate.Group.SIMPLE) { throw new AssertionError(Bug.CALCITE_461_FIXED); } @@ -654,12 +647,8 @@ public Frame decorrelateRel(LogicalAggregate rel) throws SemanticException{ newInputOutputFieldCount + i); } - relBuilder.push( - LogicalAggregate.create(newProject, - false, - newGroupSet, - null, - newAggCalls)); + relBuilder.push(newProject) + .aggregate(relBuilder.groupKey(newGroupSet, null), newAggCalls); if (!omittedConstants.isEmpty()) { final List postProjects = new ArrayList<>(relBuilder.fields()); @@ -877,17 +866,17 @@ public Frame decorrelateRel(HiveProject rel) throws SemanticException{ final List oldProjects = rel.getProjects(); final List relOutput = rel.getRowType().getFieldList(); - // LogicalProject projects the original expressions, + // Project projects the original expressions, // plus any correlated variables the input wants to pass along. final List> projects = Lists.newArrayList(); - // If this LogicalProject has correlated reference, create value generator + // If this Project has correlated reference, create value generator // and produce the correlated variables in the new output. if (cm.mapRefRelToCorRef.containsKey(rel)) { frame = decorrelateInputWithValueGenerator(rel); } - // LogicalProject projects the original expressions + // Project projects the original expressions final Map mapOldToNewOutputs = new HashMap<>(); int newPos; for (newPos = 0; newPos < oldProjects.size(); newPos++) { @@ -917,11 +906,11 @@ public Frame decorrelateRel(HiveProject rel) throws SemanticException{ corDefOutputs); } /** - * Rewrite LogicalProject. + * Rewrite Project. * * @param rel the project rel to rewrite */ - public Frame decorrelateRel(LogicalProject rel) throws SemanticException{ + public Frame decorrelateRel(Project rel) throws SemanticException{ // // Rewrite logic: // @@ -937,17 +926,17 @@ public Frame decorrelateRel(LogicalProject rel) throws SemanticException{ final List oldProjects = rel.getProjects(); final List relOutput = rel.getRowType().getFieldList(); - // LogicalProject projects the original expressions, + // Project projects the original expressions, // plus any correlated variables the input wants to pass along. final List> projects = Lists.newArrayList(); - // If this LogicalProject has correlated reference, create value generator + // If this Project has correlated reference, create value generator // and produce the correlated variables in the new output. if (cm.mapRefRelToCorRef.containsKey(rel)) { frame = decorrelateInputWithValueGenerator(rel); } - // LogicalProject projects the original expressions + // Project projects the original expressions final Map mapOldToNewOutputs = new HashMap<>(); int newPos; for (newPos = 0; newPos < oldProjects.size(); newPos++) { @@ -977,13 +966,6 @@ public Frame decorrelateRel(LogicalProject rel) throws SemanticException{ /** * Create RelNode tree that produces a list of correlated variables. - * - * @param correlations correlated variables to generate - * @param valueGenFieldOffset offset in the output that generated columns - * will start - * @param mapCorVarToOutputPos output positions for the correlated variables - * generated - * @return RelNode the root of the resultant RelNode tree */ private RelNode createValueGenerator( Iterable correlations, @@ -1039,11 +1021,9 @@ private RelNode createValueGenerator( assert newInput != null; if (!joinedInputs.contains(newInput)) { - RelNode project = - RelOptUtil.createProject( - newInput, - mapNewInputToOutputs.get(newInput)); - RelNode distinct = RelOptUtil.createDistinctRel(project); + RelNode project = RelOptUtil.createProject( + HiveRelFactories.HIVE_PROJECT_FACTORY, newInput, mapNewInputToOutputs.get(newInput)); + RelNode distinct = relBuilder.push(project).distinct().build(); RelOptCluster cluster = distinct.getCluster(); joinedInputs.add(newInput); @@ -1053,10 +1033,8 @@ private RelNode createValueGenerator( if (r == null) { r = distinct; } else { - r = - LogicalJoin.create(r, distinct, - cluster.getRexBuilder().makeLiteral(true), - ImmutableSet.of(), JoinRelType.INNER); + r = relBuilder.push(r).push(distinct) + .join(JoinRelType.INNER, cluster.getRexBuilder().makeLiteral(true)).build(); } } } @@ -1153,13 +1131,12 @@ private Frame decorrelateInputWithValueGenerator(RelNode rel) { leftInputOutputCount, corDefOutputs); - RelNode join = - LogicalJoin.create(frame.r, valueGenRel, rexBuilder.makeLiteral(true), - ImmutableSet.of(), JoinRelType.INNER); + RelNode join = relBuilder.push(frame.r).push(valueGenRel) + .join(JoinRelType.INNER, rexBuilder.makeLiteral(true)).build(); - // LogicalJoin or LogicalFilter does not change the old input ordering. All + // Join or Filter does not change the old input ordering. All // input fields from newLeftInput(i.e. the original input to the old - // LogicalFilter) are in the output and in the same position. + // Filter) are in the output and in the same position. return register(oldInput, join, frame.oldToNewOutputs, corDefOutputs); } @@ -1242,16 +1219,16 @@ public Frame decorrelateRel(HiveFilter rel) throws SemanticException { // // Rewrite logic: // - // 1. If a LogicalFilter references a correlated field in its filter - // condition, rewrite the LogicalFilter to be - // LogicalFilter - // LogicalJoin(cross product) + // 1. If a Filter references a correlated field in its filter + // condition, rewrite the Filter to be + // Filter + // Join(cross product) // OriginalFilterInput // ValueGenerator(produces distinct sets of correlated variables) // and rewrite the correlated fieldAccess in the filter condition to - // reference the LogicalJoin output. + // reference the Join output. // - // 2. If LogicalFilter does not reference correlated variables, simply + // 2. If Filter does not reference correlated variables, simply // rewrite the filter condition using new input. // @@ -1263,7 +1240,7 @@ public Frame decorrelateRel(HiveFilter rel) throws SemanticException { } Frame oldInputFrame = frame; - // If this LogicalFilter has correlated reference, create value generator + // If this Filter has correlated reference, create value generator // and produce the correlated variables in the new output. if (cm.mapRefRelToCorRef.containsKey(rel)) { frame = decorrelateInputWithValueGenerator(rel); @@ -1306,24 +1283,24 @@ public Frame decorrelateRel(HiveFilter rel) throws SemanticException { } /** - * Rewrite LogicalFilter. + * Rewrite Filter. * * @param rel the filter rel to rewrite */ - public Frame decorrelateRel(LogicalFilter rel) { + public Frame decorrelateRel(Filter rel) { // // Rewrite logic: // - // 1. If a LogicalFilter references a correlated field in its filter - // condition, rewrite the LogicalFilter to be - // LogicalFilter - // LogicalJoin(cross product) + // 1. If a Filter references a correlated field in its filter + // condition, rewrite the Filter to be + // Filter + // Join(cross product) // OriginalFilterInput // ValueGenerator(produces distinct sets of correlated variables) // and rewrite the correlated fieldAccess in the filter condition to - // reference the LogicalJoin output. + // reference the Join output. // - // 2. If LogicalFilter does not reference correlated variables, simply + // 2. If Filter does not reference correlated variables, simply // rewrite the filter condition using new input. // @@ -1334,7 +1311,7 @@ public Frame decorrelateRel(LogicalFilter rel) { return null; } - // If this LogicalFilter has correlated reference, create value generator + // If this Filter has correlated reference, create value generator // and produce the correlated variables in the new output. if (cm.mapRefRelToCorRef.containsKey(rel)) { frame = decorrelateInputWithValueGenerator(rel); @@ -1500,9 +1477,8 @@ public Frame decorrelateRel(LogicalCorrelate rel) { rightFrame.oldToNewOutputs.get(i) + newLeftFieldCount); } - newJoin = LogicalJoin.create(leftFrame.r, rightFrame.r, condition, - ImmutableSet.of(), rel.getJoinType().toJoinType()); - + newJoin = relBuilder.push(leftFrame.r).push(rightFrame.r) + .join(rel.getJoinType().toJoinType(), condition).build(); } valueGen.pop(); @@ -1564,11 +1540,11 @@ public Frame decorrelateRel(HiveJoin rel) throws SemanticException{ return register(rel, newJoin, mapOldToNewOutputs, corDefOutputs); } /** - * Rewrite LogicalJoin. + * Rewrite Join. * - * @param rel LogicalJoin + * @param rel Join */ - public Frame decorrelateRel(LogicalJoin rel) { + public Frame decorrelateRel(Join rel) { // // Rewrite logic: // @@ -1684,11 +1660,11 @@ private RexInputRef getNewForOldInputRef(RexInputRef oldInputRef) { * @param join Join * @param project Original project as the right-hand input of the join * @param nullIndicatorPos Position of null indicator - * @return the subtree with the new LogicalProject at the root + * @return the subtree with the new Project at the root */ private RelNode projectJoinOutputWithNullability( - LogicalJoin join, - LogicalProject project, + Join join, + Project project, int nullIndicatorPos) { final RelDataTypeFactory typeFactory = join.getCluster().getTypeFactory(); final RelNode left = join.getLeft(); @@ -1730,7 +1706,8 @@ private RelNode projectJoinOutputWithNullability( newProjExprs.add(Pair.of(newProjExpr, pair.right)); } - return RelOptUtil.createProject(join, newProjExprs, false); + return RelOptUtil.createProject(join, Pair.left(newProjExprs), Pair.right(newProjExprs), + false, relBuilder); } /** @@ -1741,11 +1718,11 @@ private RelNode projectJoinOutputWithNullability( * @param project the original project as the RHS input of the join * @param isCount Positions which are calls to the COUNT * aggregation function - * @return the subtree with the new LogicalProject at the root + * @return the subtree with the new Project at the root */ private RelNode aggregateCorrelatorOutput( Correlate correlate, - LogicalProject project, + Project project, Set isCount) { final RelNode left = correlate.getLeft(); final JoinRelType joinType = correlate.getJoinType().toJoinType(); @@ -1777,7 +1754,8 @@ private RelNode aggregateCorrelatorOutput( newProjects.add(Pair.of(newProjExpr, pair.right)); } - return RelOptUtil.createProject(correlate, newProjects, false); + return RelOptUtil.createProject(correlate, Pair.left(newProjects), Pair.right(newProjects), + false, relBuilder); } /** @@ -1792,8 +1770,8 @@ private RelNode aggregateCorrelatorOutput( */ private boolean checkCorVars( LogicalCorrelate correlate, - LogicalProject project, - LogicalFilter filter, + Project project, + Filter filter, List correlatedJoinKeys) { if (filter != null) { assert correlatedJoinKeys != null; @@ -1852,7 +1830,7 @@ private void removeCorVarFromTree(LogicalCorrelate correlate) { * * @param input Input relational expression * @param additionalExprs Additional expressions and names - * @return the new LogicalProject + * @return the new Project */ private RelNode createProjectWithAdditionalExprs( RelNode input, @@ -1868,7 +1846,8 @@ private RelNode createProjectWithAdditionalExprs( field.e.getName())); } projects.addAll(additionalExprs); - return RelOptUtil.createProject(input, projects, false); + return RelOptUtil.createProject(input, Pair.left(projects), Pair.right(projects), + false, relBuilder); } /* Returns an immutable map with the identity [0: 0, .., count-1: count-1]. */ @@ -2206,16 +2185,16 @@ private RexNode createCaseExpression( RemoveSingleAggregateRule() { super( operand( - LogicalAggregate.class, + Aggregate.class, operand( - LogicalProject.class, - operand(LogicalAggregate.class, any())))); + Project.class, + operand(Aggregate.class, any())))); } public void onMatch(RelOptRuleCall call) { - LogicalAggregate singleAggregate = call.rel(0); - LogicalProject project = call.rel(1); - LogicalAggregate aggregate = call.rel(2); + Aggregate singleAggregate = call.rel(0); + Project project = call.rel(1); + Aggregate aggregate = call.rel(2); // check singleAggRel is single_value agg if ((!singleAggregate.getGroupSet().isEmpty()) @@ -2241,15 +2220,11 @@ public void onMatch(RelOptRuleCall call) { // singleAggRel produces a nullable type, so create the new // projection that casts proj expr to a nullable type. final RelOptCluster cluster = project.getCluster(); - RelNode newProject = - RelOptUtil.createProject(aggregate, - ImmutableList.of( - rexBuilder.makeCast( - cluster.getTypeFactory().createTypeWithNullability( - projExprs.get(0).getType(), - true), - projExprs.get(0))), - null); + RelNode newProject = RelOptUtil.createProject(aggregate, + ImmutableList.of(rexBuilder.makeCast( + cluster.getTypeFactory().createTypeWithNullability(projExprs.get(0).getType(), true), + projExprs.get(0))), + null, false, relBuilder); call.transformTo(newProject); } } @@ -2260,16 +2235,16 @@ public void onMatch(RelOptRuleCall call) { super( operand(LogicalCorrelate.class, operand(RelNode.class, any()), - operand(LogicalAggregate.class, - operand(LogicalProject.class, + operand(Aggregate.class, + operand(Project.class, operand(RelNode.class, any()))))); } public void onMatch(RelOptRuleCall call) { final LogicalCorrelate correlate = call.rel(0); final RelNode left = call.rel(1); - final LogicalAggregate aggregate = call.rel(2); - final LogicalProject project = call.rel(3); + final Aggregate aggregate = call.rel(2); + final Project project = call.rel(3); RelNode right = call.rel(4); final RelOptCluster cluster = correlate.getCluster(); @@ -2281,8 +2256,8 @@ public void onMatch(RelOptRuleCall call) { // // CorrelateRel(left correlation, condition = true) // LeftInputRel - // LogicalAggregate (groupby (0) single_value()) - // LogicalProject-A (may reference coVar) + // Aggregate (groupby (0) single_value()) + // Project-A (may reference coVar) // RightInputRel final JoinRelType joinType = correlate.getJoinType().toJoinType(); @@ -2311,18 +2286,18 @@ public void onMatch(RelOptRuleCall call) { int nullIndicatorPos; - if ((right instanceof LogicalFilter) + if ((right instanceof Filter) && cm.mapRefRelToCorRef.containsKey(right)) { // rightInputRel has this shape: // - // LogicalFilter (references corvar) + // Filter (references corvar) // FilterInputRel // If rightInputRel is a filter and contains correlated // reference, make sure the correlated keys in the filter // condition forms a unique key of the RHS. - LogicalFilter filter = (LogicalFilter) right; + Filter filter = (Filter) right; right = filter.getInput(); assert right instanceof HepRelVertex; @@ -2342,7 +2317,7 @@ public void onMatch(RelOptRuleCall call) { // refs. These comparisons are AND'ed together. List tmpRightJoinKeys = Lists.newArrayList(); List correlatedJoinKeys = Lists.newArrayList(); - RelOptUtil.splitCorrelatedFilterCondition( + HiveRelOptUtil.splitCorrelatedFilterCondition( filter, tmpRightJoinKeys, correlatedJoinKeys, @@ -2386,8 +2361,8 @@ public void onMatch(RelOptRuleCall call) { // Change the plan to this structure. // Note that the aggregateRel is removed. // - // LogicalProject-A' (replace corvar to input ref from the LogicalJoin) - // LogicalJoin (replace corvar to input ref from LeftInputRel) + // Project-A' (replace corvar to input ref from the Join) + // Join (replace corvar to input ref from LeftInputRel) // LeftInputRel // RightInputRel(oreviously FilterInputRel) @@ -2410,11 +2385,11 @@ public void onMatch(RelOptRuleCall call) { // Change the plan to this structure. // - // LogicalProject-A' (replace corvar to input ref from LogicalJoin) - // LogicalJoin (left, condition = true) + // Project-A' (replace corvar to input ref from Join) + // Join (left, condition = true) // LeftInputRel - // LogicalAggregate(groupby(0), single_value(0), s_v(1)....) - // LogicalProject-B (everything from input plus literal true) + // Aggregate(groupby(0), single_value(0), s_v(1)....) + // Project-B (everything from input plus literal true) // ProjInputRel // make the new projRel to provide a null indicator @@ -2426,7 +2401,7 @@ public void onMatch(RelOptRuleCall call) { // make the new aggRel right = - RelOptUtil.createSingleValueAggRel(cluster, right); + HiveRelOptUtil.createSingleValueAggRel(cluster, right, HiveRelFactories.HIVE_AGGREGATE_FACTORY); // The last field: // single_value(true) @@ -2439,9 +2414,8 @@ public void onMatch(RelOptRuleCall call) { } // make the new join rel - LogicalJoin join = - LogicalJoin.create(left, right, joinCond, - ImmutableSet.of(), joinType); + Join join = (Join) relBuilder.push(left).push(right) + .join(joinType, joinCond).build(); RelNode newProject = projectJoinOutputWithNullability(join, project, nullIndicatorPos); @@ -2459,18 +2433,18 @@ public void onMatch(RelOptRuleCall call) { super( operand(LogicalCorrelate.class, operand(RelNode.class, any()), - operand(LogicalProject.class, - operand(LogicalAggregate.class, null, Aggregate.IS_SIMPLE, - operand(LogicalProject.class, + operand(Project.class, + operand(Aggregate.class, null, Aggregate.IS_SIMPLE, + operand(Project.class, operand(RelNode.class, any())))))); } public void onMatch(RelOptRuleCall call) { final LogicalCorrelate correlate = call.rel(0); final RelNode left = call.rel(1); - final LogicalProject aggOutputProject = call.rel(2); - final LogicalAggregate aggregate = call.rel(3); - final LogicalProject aggInputProject = call.rel(4); + final Project aggOutputProject = call.rel(2); + final Aggregate aggregate = call.rel(3); + final Project aggInputProject = call.rel(4); RelNode right = call.rel(5); final RelOptCluster cluster = correlate.getCluster(); @@ -2482,9 +2456,9 @@ public void onMatch(RelOptRuleCall call) { // // CorrelateRel(left correlation, condition = true) // LeftInputRel - // LogicalProject-A (a RexNode) - // LogicalAggregate (groupby (0), agg0(), agg1()...) - // LogicalProject-B (references coVar) + // Project-A (a RexNode) + // Aggregate (groupby (0), agg0(), agg1()...) + // Project-B (references coVar) // rightInputRel // check aggOutputProject projects only one expression @@ -2523,13 +2497,13 @@ public void onMatch(RelOptRuleCall call) { } } - if ((right instanceof LogicalFilter) + if ((right instanceof Filter) && cm.mapRefRelToCorRef.containsKey(right)) { // rightInputRel has this shape: // - // LogicalFilter (references corvar) + // Filter (references corvar) // FilterInputRel - LogicalFilter filter = (LogicalFilter) right; + Filter filter = (Filter) right; right = filter.getInput(); assert right instanceof HepRelVertex; @@ -2550,7 +2524,7 @@ public void onMatch(RelOptRuleCall call) { // expressions. These comparisons are AND'ed together. List rightJoinKeys = Lists.newArrayList(); List tmpCorrelatedJoinKeys = Lists.newArrayList(); - RelOptUtil.splitCorrelatedFilterCondition( + HiveRelOptUtil.splitCorrelatedFilterCondition( filter, rightJoinKeys, tmpCorrelatedJoinKeys, @@ -2600,21 +2574,21 @@ public void onMatch(RelOptRuleCall call) { // // CorrelateRel(left correlation, condition = true) // LeftInputRel - // LogicalProject-A (a RexNode) - // LogicalAggregate (groupby(0), agg0(),agg1()...) - // LogicalProject-B (may reference coVar) - // LogicalFilter (references corVar) + // Project-A (a RexNode) + // Aggregate (groupby(0), agg0(),agg1()...) + // Project-B (may reference coVar) + // Filter (references corVar) // RightInputRel (no correlated reference) // // to this plan: // - // LogicalProject-A' (all gby keys + rewritten nullable ProjExpr) - // LogicalAggregate (groupby(all left input refs) + // Project-A' (all gby keys + rewritten nullable ProjExpr) + // Aggregate (groupby(all left input refs) // agg0(rewritten expression), // agg1()...) - // LogicalProject-B' (rewriten original projected exprs) - // LogicalJoin(replace corvar w/ input ref from LeftInputRel) + // Project-B' (rewriten original projected exprs) + // Join(replace corvar w/ input ref from LeftInputRel) // LeftInputRel // RightInputRel // @@ -2626,14 +2600,14 @@ public void onMatch(RelOptRuleCall call) { // projection list from the RHS for simplicity to avoid // searching for non-null fields. // - // LogicalProject-A' (all gby keys + rewritten nullable ProjExpr) - // LogicalAggregate (groupby(all left input refs), + // Project-A' (all gby keys + rewritten nullable ProjExpr) + // Aggregate (groupby(all left input refs), // count(nullIndicator), other aggs...) - // LogicalProject-B' (all left input refs plus + // Project-B' (all left input refs plus // the rewritten original projected exprs) - // LogicalJoin(replace corvar to input ref from LeftInputRel) + // Join(replace corvar to input ref from LeftInputRel) // LeftInputRel - // LogicalProject (everything from RightInputRel plus + // Project (everything from RightInputRel plus // the nullIndicator "true") // RightInputRel // @@ -2668,20 +2642,20 @@ public void onMatch(RelOptRuleCall call) { // // CorrelateRel(left correlation, condition = true) // LeftInputRel - // LogicalProject-A (a RexNode) - // LogicalAggregate (groupby(0), agg0(), agg1()...) - // LogicalProject-B (references coVar) + // Project-A (a RexNode) + // Aggregate (groupby(0), agg0(), agg1()...) + // Project-B (references coVar) // RightInputRel (no correlated reference) // // to this plan: // - // LogicalProject-A' (all gby keys + rewritten nullable ProjExpr) - // LogicalAggregate (groupby(all left input refs) + // Project-A' (all gby keys + rewritten nullable ProjExpr) + // Aggregate (groupby(all left input refs) // agg0(rewritten expression), // agg1()...) - // LogicalProject-B' (rewriten original projected exprs) - // LogicalJoin (LOJ cond = true) + // Project-B' (rewriten original projected exprs) + // Join (LOJ cond = true) // LeftInputRel // RightInputRel // @@ -2693,14 +2667,14 @@ public void onMatch(RelOptRuleCall call) { // projection list from the RHS for simplicity to avoid // searching for non-null fields. // - // LogicalProject-A' (all gby keys + rewritten nullable ProjExpr) - // LogicalAggregate (groupby(all left input refs), + // Project-A' (all gby keys + rewritten nullable ProjExpr) + // Aggregate (groupby(all left input refs), // count(nullIndicator), other aggs...) - // LogicalProject-B' (all left input refs plus + // Project-B' (all left input refs plus // the rewritten original projected exprs) - // LogicalJoin(replace corvar to input ref from LeftInputRel) + // Join(replace corvar to input ref from LeftInputRel) // LeftInputRel - // LogicalProject (everything from RightInputRel plus + // Project (everything from RightInputRel plus // the nullIndicator "true") // RightInputRel } else { @@ -2718,9 +2692,8 @@ public void onMatch(RelOptRuleCall call) { Pair.of(rexBuilder.makeLiteral(true), "nullIndicator"))); - LogicalJoin join = - LogicalJoin.create(left, right, joinCond, - ImmutableSet.of(), joinType); + Join join = (Join) relBuilder.push(left).push(right) + .join(joinType, joinCond).build(); // To the consumer of joinOutputProjRel, nullIndicator is located // at the end @@ -2754,11 +2727,8 @@ public void onMatch(RelOptRuleCall call) { joinOutputProjects.add( rexBuilder.makeInputRef(join, nullIndicatorPos)); - RelNode joinOutputProject = - RelOptUtil.createProject( - join, - joinOutputProjects, - null); + RelNode joinOutputProject = RelOptUtil.createProject( + join, joinOutputProjects, null, false, relBuilder); // nullIndicator is now at a different location in the output of // the join @@ -2793,12 +2763,9 @@ public void onMatch(RelOptRuleCall call) { ImmutableBitSet groupSet = ImmutableBitSet.range(groupCount); - LogicalAggregate newAggregate = - LogicalAggregate.create(joinOutputProject, - false, - groupSet, - null, - newAggCalls); + Aggregate newAggregate = + (Aggregate) relBuilder.push(joinOutputProject) + .aggregate(relBuilder.groupKey(groupSet, null), newAggCalls).build(); List newAggOutputProjectList = Lists.newArrayList(); for (int i : groupSet) { @@ -2815,11 +2782,8 @@ public void onMatch(RelOptRuleCall call) { true), newAggOutputProjects)); - RelNode newAggOutputProject = - RelOptUtil.createProject( - newAggregate, - newAggOutputProjectList, - null); + RelNode newAggOutputProject = RelOptUtil.createProject( + newAggregate, newAggOutputProjectList, null, false, relBuilder); call.transformTo(newAggOutputProject); @@ -2844,19 +2808,19 @@ public void onMatch(RelOptRuleCall call) { flavor ? operand(LogicalCorrelate.class, operand(RelNode.class, any()), - operand(LogicalProject.class, - operand(LogicalAggregate.class, any()))) + operand(Project.class, + operand(Aggregate.class, any()))) : operand(LogicalCorrelate.class, operand(RelNode.class, any()), - operand(LogicalAggregate.class, any()))); + operand(Aggregate.class, any()))); this.flavor = flavor; } public void onMatch(RelOptRuleCall call) { final LogicalCorrelate correlate = call.rel(0); final RelNode left = call.rel(1); - final LogicalProject aggOutputProject; - final LogicalAggregate aggregate; + final Project aggOutputProject; + final Aggregate aggregate; if (flavor) { aggOutputProject = call.rel(2); aggregate = call.rel(3); @@ -2870,11 +2834,8 @@ public void onMatch(RelOptRuleCall call) { for (int i = 0; i < fields.size(); i++) { projects.add(RexInputRef.of2(projects.size(), fields)); } - aggOutputProject = - (LogicalProject) RelOptUtil.createProject( - aggregate, - projects, - false); + aggOutputProject = (Project) RelOptUtil.createProject( + aggregate, Pair.left(projects), Pair.right(projects), false, relBuilder); } onMatch2(call, correlate, left, aggOutputProject, aggregate); } @@ -2883,8 +2844,8 @@ private void onMatch2( RelOptRuleCall call, LogicalCorrelate correlate, RelNode leftInput, - LogicalProject aggOutputProject, - LogicalAggregate aggregate) { + Project aggOutputProject, + Aggregate aggregate) { if (generatedCorRels.contains(correlate)) { // This correlator was generated by a previous invocation of // this rule. No further work to do. @@ -2899,8 +2860,8 @@ private void onMatch2( // // CorrelateRel(left correlation, condition = true) // LeftInputRel - // LogicalProject-A (a RexNode) - // LogicalAggregate (groupby (0), agg0(), agg1()...) + // Project-A (a RexNode) + // Aggregate (groupby (0), agg0(), agg1()...) // check aggOutputProj projects only one expression List aggOutputProjExprs = aggOutputProject.getProjects(); @@ -2940,7 +2901,7 @@ private void onMatch2( // replacing references to count() with case statement) // Correlator(left correlation, condition = true) // LeftInputRel - // LogicalAggregate (groupby (0), agg0(), agg1()...) + // Aggregate (groupby (0), agg0(), agg1()...) // LogicalCorrelate newCorrelate = LogicalCorrelate.create(leftInput, aggregate, @@ -3179,24 +3140,10 @@ public RelNode visit(HiveUnion rel) { mightRequireValueGen = true; return rel; } - public RelNode visit(LogicalUnion rel) { - mightRequireValueGen = true; - return rel; - } - public RelNode visit(LogicalIntersect rel) { - mightRequireValueGen = true; - return rel; - } - public RelNode visit(HiveIntersect rel) { mightRequireValueGen = true; return rel; } - - @Override public RelNode visit(LogicalJoin rel) { - mightRequireValueGen = true; - return rel; - } @Override public RelNode visit(HiveProject rel) { if(!(hasRexOver(((HiveProject)rel).getProjects()))) { mightRequireValueGen = false; @@ -3206,15 +3153,6 @@ public RelNode visit(HiveIntersect rel) { return rel; } } - @Override public RelNode visit(LogicalProject rel) { - if(!(hasRexOver(((LogicalProject)rel).getProjects()))) { - mightRequireValueGen = false; - return super.visit(rel); - } else { - mightRequireValueGen = true; - return rel; - } - } @Override public RelNode visit(HiveAggregate rel) { // if there are aggregate functions or grouping sets we will need // value generator @@ -3228,17 +3166,6 @@ public RelNode visit(HiveIntersect rel) { return rel; } } - @Override public RelNode visit(LogicalAggregate rel) { - if(rel.getAggCallList().isEmpty() && !rel.indicator) { - this.mightRequireValueGen = false; - return super.visit(rel); - } else { - // need to reset to true in case previous aggregate/project - // has set it to false - this.mightRequireValueGen = true; - return rel; - } - } @Override public RelNode visit(LogicalCorrelate rel) { // this means we are hitting nested subquery so don't // need to go further @@ -3279,16 +3206,6 @@ CorelMap build(RelNode rel) { mapFieldAccessToCorVar); } - @Override public RelNode visit(LogicalJoin join) { - try { - Stacks.push(stack, join); - join.getCondition().accept(rexVisitor(join)); - } finally { - Stacks.pop(stack, join); - } - return visitJoin(join); - } - public RelNode visit(HiveJoin join) { try { Stacks.push(stack, join); @@ -3329,6 +3246,7 @@ public RelNode visit(final HiveProject project) { } return super.visit(project); } + public RelNode visit(final HiveFilter filter) { try { Stacks.push(stack, filter); @@ -3338,27 +3256,6 @@ public RelNode visit(final HiveFilter filter) { } return super.visit(filter); } - @Override public RelNode visit(final LogicalFilter filter) { - try { - Stacks.push(stack, filter); - filter.getCondition().accept(rexVisitor(filter)); - } finally { - Stacks.pop(stack, filter); - } - return super.visit(filter); - } - - @Override public RelNode visit(LogicalProject project) { - try { - Stacks.push(stack, project); - for (RexNode node : project.getProjects()) { - node.accept(rexVisitor(project)); - } - } finally { - Stacks.pop(stack, project); - } - return super.visit(project); - } private RexVisitorImpl rexVisitor(final RelNode rel) { return new RexVisitorImpl(true) { diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 872f9c98a7..1af08abf87 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -2287,27 +2287,27 @@ Stage-0 limit:-1 Stage-1 Reducer 3 llap - File Output Operator [FS_26] - Select Operator [SEL_25] (rows=13 width=223) + File Output Operator [FS_24] + Select Operator [SEL_23] (rows=13 width=223) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_24] (rows=13 width=231) - predicate:(not CASE WHEN ((_col4 = 0L)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) - Merge Join Operator [MERGEJOIN_32] (rows=26 width=230) - Conds:RS_21._col0, _col1=RS_22._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col8"] + Filter Operator [FIL_22] (rows=13 width=231) + predicate:CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END + Merge Join Operator [MERGEJOIN_30] (rows=26 width=230) + Conds:RS_19._col0, _col1=RS_20._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col8"] <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] + SHUFFLE [RS_19] PartitionCols:_col0, _col1 - Merge Join Operator [MERGEJOIN_31] (rows=26 width=229) - Conds:RS_18._col1=RS_19._col0(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5"] + Merge Join Operator [MERGEJOIN_29] (rows=26 width=229) + Conds:RS_16._col1=RS_17._col0(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5"] <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_18] + SHUFFLE [RS_16] PartitionCols:_col1 Select Operator [SEL_1] (rows=26 width=223) Output:["_col0","_col1","_col2"] TableScan [TS_0] (rows=26 width=223) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] <-Reducer 4 [ONE_TO_ONE_EDGE] llap - FORWARD [RS_19] + FORWARD [RS_17] PartitionCols:_col0 Group By Operator [GBY_7] (rows=2 width=114) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 @@ -2318,28 +2318,26 @@ Stage-0 Output:["_col0","_col1","_col2"],aggregations:["count()","count(p_name)"],keys:p_mfgr Select Operator [SEL_4] (rows=8 width=223) Output:["p_name","p_mfgr"] - Filter Operator [FIL_29] (rows=8 width=223) + Filter Operator [FIL_27] (rows=8 width=223) predicate:((p_size < 10) and p_mfgr is not null) Please refer to the previous TableScan [TS_0] <-Reducer 5 [ONE_TO_ONE_EDGE] llap - FORWARD [RS_22] + FORWARD [RS_20] PartitionCols:_col0, _col1 - Select Operator [SEL_17] (rows=4 width=223) + Select Operator [SEL_15] (rows=4 width=223) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_16] (rows=4 width=219) - predicate:_col0 is not null - Group By Operator [GBY_14] (rows=4 width=219) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] - PartitionCols:_col0, _col1 - Group By Operator [GBY_12] (rows=4 width=219) - Output:["_col0","_col1"],keys:p_name, p_mfgr - Select Operator [SEL_11] (rows=8 width=223) - Output:["p_name","p_mfgr"] - Filter Operator [FIL_30] (rows=8 width=223) - predicate:((p_size < 10) and p_mfgr is not null) - Please refer to the previous TableScan [TS_0] + Group By Operator [GBY_14] (rows=4 width=219) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Map 1 [SIMPLE_EDGE] llap + SHUFFLE [RS_13] + PartitionCols:_col0, _col1 + Group By Operator [GBY_12] (rows=4 width=219) + Output:["_col0","_col1"],keys:p_name, p_mfgr + Select Operator [SEL_11] (rows=8 width=223) + Output:["p_name","p_mfgr"] + Filter Operator [FIL_28] (rows=8 width=223) + predicate:((p_size < 10) and p_mfgr is not null and p_name is not null) + Please refer to the previous TableScan [TS_0] PREHOOK: query: explain select p_name, p_size from @@ -2451,24 +2449,24 @@ Stage-0 limit:-1 Stage-1 Reducer 5 llap - File Output Operator [FS_37] - Select Operator [SEL_36] (rows=3 width=106) + File Output Operator [FS_36] + Select Operator [SEL_35] (rows=2 width=106) Output:["_col0","_col1"] <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_35] - Select Operator [SEL_34] (rows=3 width=106) + SHUFFLE [RS_34] + Select Operator [SEL_33] (rows=2 width=106) Output:["_col0","_col1"] - Filter Operator [FIL_33] (rows=3 width=119) - predicate:(not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) - Merge Join Operator [MERGEJOIN_44] (rows=5 width=114) - Conds:RS_30._col0, _col1=RS_31._col0, _col1(Left Outer),Output:["_col0","_col1","_col3","_col4","_col7"] + Filter Operator [FIL_32] (rows=2 width=116) + predicate:CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END + Merge Join Operator [MERGEJOIN_43] (rows=5 width=114) + Conds:RS_29._col0, _col1=RS_30._col0, _col1(Left Outer),Output:["_col0","_col1","_col3","_col4","_col7"] <-Reducer 3 [SIMPLE_EDGE] llap - SHUFFLE [RS_30] + SHUFFLE [RS_29] PartitionCols:_col0, _col1 - Merge Join Operator [MERGEJOIN_43] (rows=5 width=112) - Conds:RS_27._col1=RS_28._col0(Left Outer),Output:["_col0","_col1","_col3","_col4"] + Merge Join Operator [MERGEJOIN_42] (rows=5 width=112) + Conds:RS_26._col1=RS_27._col0(Left Outer),Output:["_col0","_col1","_col3","_col4"] <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_27] + SHUFFLE [RS_26] PartitionCols:_col1 Group By Operator [GBY_4] (rows=5 width=106) Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 @@ -2482,7 +2480,7 @@ Stage-0 TableScan [TS_0] (rows=26 width=106) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] <-Reducer 7 [ONE_TO_ONE_EDGE] llap - FORWARD [RS_28] + FORWARD [RS_27] PartitionCols:_col0 Group By Operator [GBY_16] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 @@ -2493,7 +2491,7 @@ Stage-0 Output:["_col0","_col1","_col2"],aggregations:["count()","count(_col0)"],keys:_col1 Select Operator [SEL_12] (rows=1 width=114) Output:["_col0","_col1"] - Filter Operator [FIL_40] (rows=1 width=114) + Filter Operator [FIL_39] (rows=1 width=114) predicate:(((_col2 - _col1) > 600.0D) and _col1 is not null) Group By Operator [GBY_10] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 @@ -2504,24 +2502,22 @@ Stage-0 Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr Please refer to the previous Select Operator [SEL_1] <-Reducer 8 [SIMPLE_EDGE] llap - SHUFFLE [RS_31] + SHUFFLE [RS_30] PartitionCols:_col0, _col1 - Select Operator [SEL_26] (rows=1 width=110) + Select Operator [SEL_25] (rows=1 width=110) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_25] (rows=1 width=110) - predicate:_col0 is not null - Select Operator [SEL_24] (rows=1 width=110) - Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=1 width=114) - predicate:(((_col2 - _col1) > 600.0D) and _col1 is not null) - Group By Operator [GBY_22] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_21] - PartitionCols:_col0 - Group By Operator [GBY_20] (rows=5 width=114) - Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr - Please refer to the previous TableScan [TS_0] + Filter Operator [FIL_40] (rows=1 width=114) + predicate:(((_col2 - _col1) > 600.0D) and _col1 is not null) + Group By Operator [GBY_23] (rows=5 width=114) + Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0 + <-Map 1 [SIMPLE_EDGE] llap + SHUFFLE [RS_22] + PartitionCols:_col0 + Group By Operator [GBY_21] (rows=5 width=114) + Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr + Filter Operator [FIL_41] (rows=26 width=106) + predicate:p_mfgr is not null + Please refer to the previous TableScan [TS_0] PREHOOK: query: explain select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out b/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out index 6ba8811631..5e18f33f67 100644 --- a/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby_groupingset_bug.q.out @@ -230,9 +230,9 @@ Stage-0 SHUFFLE [RS_139] PartitionCols:_col0 Group By Operator [GBY_138] (rows=1 width=12) - Output:["_col0","_col1"],aggregations:["count()"],keys:_col9 + Output:["_col0","_col1"],aggregations:["count()"],keys:_col8 Map Join Operator [MAPJOIN_137] (rows=5185194 width=4) - Conds:MAPJOIN_136._col7=RS_130._col0(Inner),Output:["_col9"] + Conds:MAPJOIN_136._col6=RS_130._col0(Inner),Output:["_col8"] <-Map 7 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_130] PartitionCols:_col0 @@ -271,17 +271,17 @@ Stage-0 Output:["_col0"] Please refer to the previous Map Join Operator [MAPJOIN_129] <-Map Join Operator [MAPJOIN_136] (rows=370371 width=4) - Conds:RS_30._col0=SEL_135._col0(Inner),Output:["_col7"] + Conds:RS_30._col0=SEL_135._col0(Inner),Output:["_col6"] <-Map 1 [BROADCAST_EDGE] llap BROADCAST [RS_30] PartitionCols:_col0 Map Join Operator [MAPJOIN_100] (rows=6 width=228) - Conds:SEL_2._col1=RS_113._col2(Inner),Output:["_col0","_col2","_col3","_col4"],residual filter predicates:{(_col2 > CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END)} + Conds:SEL_2._col1=RS_113._col1(Inner),Output:["_col0","_col2","_col3"],residual filter predicates:{(_col2 > _col3)} <-Reducer 3 [BROADCAST_EDGE] vectorized, llap BROADCAST [RS_113] - PartitionCols:_col2 - Select Operator [SEL_112] (rows=1 width=201) - Output:["_col0","_col1","_col2"] + PartitionCols:_col1 + Select Operator [SEL_112] (rows=1 width=197) + Output:["_col0","_col1"] Group By Operator [GBY_111] (rows=1 width=197) Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0 <-Map 2 [SIMPLE_EDGE] vectorized, llap @@ -296,7 +296,7 @@ Stage-0 <-Select Operator [SEL_2] (rows=18 width=201) Output:["_col0","_col1","_col2"] Filter Operator [FIL_57] (rows=18 width=201) - predicate:i_item_sk is not null + predicate:(i_category is not null and i_item_sk is not null) TableScan [TS_0] (rows=18 width=201) default@x1_item,i,Tbl:COMPLETE,Col:COMPLETE,Output:["i_item_sk","i_category","i_current_price"] <-Select Operator [SEL_135] (rows=123457 width=8) diff --git a/ql/src/test/results/clientpositive/llap/lineage3.q.out b/ql/src/test/results/clientpositive/llap/lineage3.q.out index 4c933a97af..1993c23025 100644 --- a/ql/src/test/results/clientpositive/llap/lineage3.q.out +++ b/ql/src/test/results/clientpositive/llap/lineage3.q.out @@ -178,7 +178,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"tez","database":"default","hash":"94e9cc0a67801fe1503a3cb0c5029d59","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0D)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"94e9cc0a67801fe1503a3cb0c5029d59","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0D)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"b.ctinyint is not null","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 311 val_311 Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select key, value from src1 @@ -196,7 +196,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"tez","database":"default","hash":"723e79692e1de404c4ffb702097586da","queryText":"select * from src1 a\nwhere not exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0D)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"},{"sources":[],"targets":[0,1],"expression":"true is null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} +{"version":"1.0","engine":"tez","database":"default","hash":"723e79692e1de404c4ffb702097586da","queryText":"select * from src1 a\nwhere not exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > 300.0D)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(a.key = a.key)","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"b.ctinyint is not null","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(UDFToDouble((UDFToInteger(b.ctinyint) + 300)) = UDFToDouble(a.key))","edgeType":"PREDICATE"},{"sources":[],"targets":[0,1],"expression":"true is null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 369 401 val_401 406 val_406 diff --git a/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out b/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out index eba2a96950..841a5c6ea7 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out @@ -1573,11 +1573,11 @@ POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@src_null POSTHOOK: Lineage: src_null.key SCRIPT [] POSTHOOK: Lineage: src_null.value EXPRESSION [] +Warning: Map Join MAPJOIN[121][bigTable=?] in task 'Map 1' is a cross product Warning: Map Join MAPJOIN[122][bigTable=?] in task 'Map 1' is a cross product Warning: Map Join MAPJOIN[123][bigTable=?] in task 'Map 1' is a cross product -Warning: Map Join MAPJOIN[124][bigTable=?] in task 'Map 1' is a cross product -Warning: Shuffle Join MERGEJOIN[125][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[126][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[124][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[125][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 6' is a cross product PREHOOK: query: explain select key, value, count(*) from src_null b @@ -1854,7 +1854,7 @@ STAGE PLANS: 1 Reducer 8 Statistics: Num rows: 1 Data size: 668 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col4 = 0L)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col2 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col2 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 1 Data size: 668 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint) @@ -1982,20 +1982,23 @@ STAGE PLANS: expressions: _col0 (type: string), _col2 (type: bigint) outputColumnNames: _col0, _col2 Statistics: Num rows: 1 Data size: 922 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col2 (type: bigint) - outputColumnNames: _col1, _col2 + Filter Operator + predicate: _col2 is not null (type: boolean) Statistics: Num rows: 1 Data size: 922 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col1 (type: string), _col2 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 + Select Operator + expressions: _col0 (type: string), _col2 (type: bigint) + outputColumnNames: _col1, _col2 Statistics: Num rows: 1 Data size: 922 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Group By Operator + keys: _col1 (type: string), _col2 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 922 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 922 Basic stats: COMPLETE Column stats: NONE Reducer 8 Execution mode: vectorized, llap Reduce Operator Tree: @@ -2005,22 +2008,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 922 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: bigint), _col0 (type: string) - outputColumnNames: _col0, _col1 + expressions: _col1 (type: bigint), _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 922 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: bigint) Statistics: Num rows: 1 Data size: 922 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 922 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: bigint) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 922 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Reducer 9 Execution mode: vectorized, llap Reduce Operator Tree: @@ -2049,11 +2045,11 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Map Join MAPJOIN[121][bigTable=?] in task 'Map 1' is a cross product Warning: Map Join MAPJOIN[122][bigTable=?] in task 'Map 1' is a cross product Warning: Map Join MAPJOIN[123][bigTable=?] in task 'Map 1' is a cross product -Warning: Map Join MAPJOIN[124][bigTable=?] in task 'Map 1' is a cross product -Warning: Shuffle Join MERGEJOIN[125][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product -Warning: Shuffle Join MERGEJOIN[126][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 6' is a cross product +Warning: Shuffle Join MERGEJOIN[124][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[125][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 6' is a cross product PREHOOK: query: select key, value, count(*) from src_null b where NOT EXISTS (select key from src_null where src_null.value <> b.value) diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out index c573b8c123..29c94997a7 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -1667,12 +1667,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: + Reducer 10 <- Reducer 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) - Reducer 7 <- Map 10 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 4 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 7 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 7 (SIMPLE_EDGE) + Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1695,7 +1696,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Map 10 + Map 11 Map Operator Tree: TableScan alias: pp @@ -1717,6 +1718,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 5 @@ -1761,8 +1767,39 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) + Filter Operator + predicate: (p_brand is not null and p_container is not null and p_type is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string), p_container (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: string) + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs + Reducer 10 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: boolean) Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -1809,7 +1846,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col14 Statistics: Num rows: 16 Data size: 3891 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col10 = 0L)) THEN (false) WHEN (_col10 is null) THEN (false) WHEN (_col14 is not null) THEN (true) WHEN (_col3 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col10 = 0L)) THEN (true) WHEN (_col10 is null) THEN (true) WHEN (_col14 is not null) THEN (false) WHEN (_col3 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 8 Data size: 1945 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) @@ -1845,16 +1882,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 7 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Group By Operator - keys: _col2 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE Reducer 8 Execution mode: vectorized, llap Reduce Operator Tree: @@ -1871,26 +1898,26 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 9 - Execution mode: vectorized, llap + Execution mode: llap Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 _col1 (type: string), _col0 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 14 Data size: 2744 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col2 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -2124,12 +2151,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: + Reducer 10 <- Reducer 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) - Reducer 7 <- Map 10 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 4 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 7 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 7 (SIMPLE_EDGE) + Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2152,7 +2180,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col7 (type: double), _col8 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Map 10 + Map 11 Map Operator Tree: TableScan alias: pp @@ -2174,6 +2202,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 5 @@ -2218,8 +2251,39 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) + Filter Operator + predicate: (p_brand is not null and p_container is not null and p_type is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_brand (type: string), p_type (type: string), p_container (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: string) + Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs + Reducer 10 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: boolean) Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -2266,7 +2330,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col14 Statistics: Num rows: 16 Data size: 5484 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col10 = 0L)) THEN (false) WHEN (_col10 is null) THEN (false) WHEN (_col14 is not null) THEN (true) WHEN (_col3 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col10 = 0L)) THEN (true) WHEN (_col10 is null) THEN (true) WHEN (_col14 is not null) THEN (false) WHEN (_col3 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 8 Data size: 2742 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) @@ -2302,16 +2366,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 7 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Group By Operator - keys: _col2 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE Reducer 8 Execution mode: vectorized, llap Reduce Operator Tree: @@ -2328,26 +2382,26 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 9 - Execution mode: vectorized, llap + Execution mode: llap Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 _col1 (type: string), _col0 (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 14 Data size: 2744 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col2 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -2475,6 +2529,9 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 260 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Filter Operator + predicate: (p_size is not null and p_type is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_type (type: string), p_size (type: int) mode: hash @@ -2533,7 +2590,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col14 Statistics: Num rows: 16 Data size: 3891 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col10 = 0L)) THEN (false) WHEN (_col10 is null) THEN (false) WHEN (_col14 is not null) THEN (true) WHEN (_col3 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col10 = 0L)) THEN (true) WHEN (_col10 is null) THEN (true) WHEN (_col14 is not null) THEN (false) WHEN (_col3 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 8 Data size: 1945 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) @@ -2569,19 +2626,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: string), _col1 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 7270564355..9281e25e6d 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -388,7 +388,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col8 Statistics: Num rows: 26 Data size: 5994 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col4 = 0L)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 13 Data size: 3007 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: int) @@ -492,7 +492,7 @@ STAGE PLANS: isPivotResult: true Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (rank_window_0 <= 2) (type: boolean) + predicate: ((rank_window_0 <= 2) and _col1 is not null) (type: boolean) Statistics: Num rows: 8 Data size: 3928 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col2 (type: string) @@ -516,19 +516,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator @@ -932,17 +929,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col4 Statistics: Num rows: 26 Data size: 5886 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (sq_count_check(_col4, true) > 0) (type: boolean) + predicate: (sq_count_check(CASE WHEN (_col4 is null) THEN (0) ELSE (_col4) END, true) > 0) (type: boolean) Statistics: Num rows: 8 Data size: 1816 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 1816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1784 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 8 Data size: 1816 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 1784 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string), _col2 (type: int) Reducer 3 Execution mode: llap @@ -953,14 +950,14 @@ STAGE PLANS: keys: 0 _col1 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col6, _col7 + outputColumnNames: _col0, _col1, _col2, _col4, _col5 Statistics: Num rows: 8 Data size: 1912 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string), _col2 (type: int) sort order: ++ Map-reduce partition columns: _col1 (type: string), _col2 (type: int) Statistics: Num rows: 8 Data size: 1912 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col6 (type: bigint), _col7 (type: bigint) + value expressions: _col0 (type: string), _col4 (type: bigint), _col5 (type: bigint) Reducer 4 Execution mode: llap Reduce Operator Tree: @@ -970,10 +967,10 @@ STAGE PLANS: keys: 0 _col1 (type: string), _col2 (type: int) 1 _col1 (type: string), _col0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col6, _col7, _col10 + outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col8 Statistics: Num rows: 8 Data size: 1924 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col6 = 0L)) THEN (false) WHEN (_col6 is null) THEN (false) WHEN (_col10 is not null) THEN (true) WHEN (_col2 is null) THEN (null) WHEN ((_col7 < _col6)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col2 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 4 Data size: 964 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: string), _col0 (type: string), _col2 (type: int) @@ -1161,23 +1158,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int), _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: int) - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator @@ -1649,6 +1642,9 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Filter Operator + predicate: (((p_size * p_size) <> 340) and p_size is not null and p_type is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: (p_size * p_size) (type: int), p_type (type: string) outputColumnNames: _col0, _col1 @@ -1694,7 +1690,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col14 Statistics: Num rows: 26 Data size: 16394 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col10 = 0L)) THEN (false) WHEN (_col10 is null) THEN (false) WHEN (_col14 is not null) THEN (true) WHEN (_col5 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col10 = 0L)) THEN (true) WHEN (_col10 is null) THEN (true) WHEN (_col14 is not null) THEN (false) WHEN (_col5 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 13 Data size: 8207 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) @@ -1731,22 +1727,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col0 (type: string) - outputColumnNames: _col0, _col1 + expressions: _col1 (type: int), _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator @@ -2256,6 +2245,9 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1781 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Filter Operator + predicate: (p_name is not null and p_partkey is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_partkey (type: int), p_name (type: string) mode: hash @@ -2273,16 +2265,19 @@ STAGE PLANS: TableScan alias: e Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: p_size is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 10 @@ -2328,7 +2323,7 @@ STAGE PLANS: outputColumnNames: _col1, _col3, _col4, _col7 Statistics: Num rows: 26 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN ((_col1 + 100) is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN ((_col1 + 100) is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 13 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 13 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE @@ -2615,12 +2610,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: + Reducer 10 <- Map 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 5 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) + Reducer 8 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2645,35 +2641,72 @@ STAGE PLANS: TableScan alias: p Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 + Filter Operator + predicate: (p_partkey is not null and p_size is not null) (type: boolean) Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + Reduce Output Operator + key expressions: _col2 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Filter Operator + predicate: (p_name is not null and p_partkey is not null and p_size is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col0 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Map 8 + Map 9 Map Operator Tree: TableScan alias: part Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: p_size is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs + Reducer 10 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), (_col0 + 121150) (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), (_col0 + 121150) (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -2703,7 +2736,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col11, _col14 Statistics: Num rows: 26 Data size: 16250 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col10 = 0L)) THEN (false) WHEN (_col10 is null) THEN (false) WHEN (_col14 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col10 = 0L)) THEN (true) WHEN (_col10 is null) THEN (true) WHEN (_col14 is not null) THEN (false) WHEN (_col1 is null) THEN (null) WHEN ((_col11 < _col10)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 13 Data size: 8127 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) @@ -2739,16 +2772,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Group By Operator - keys: _col1 (type: string), _col3 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE Reducer 6 Execution mode: vectorized, llap Reduce Operator Tree: @@ -2765,6 +2788,27 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col2 (type: int), _col0 (type: int) + 1 _col0 (type: int), (_col0 + 121150) (type: int) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col1 (type: string), _col3 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 8 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -2772,32 +2816,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: string), _col1 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) - Reducer 9 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), (_col0 + 121150) (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator @@ -2893,6 +2921,9 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 13 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) + Filter Operator + predicate: (p_name is not null and p_partkey is not null and p_size is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_partkey (type: int), p_name (type: string), p_size (type: int) mode: hash @@ -2934,7 +2965,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col16 Statistics: Num rows: 26 Data size: 16374 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col11 = 0L)) THEN (false) WHEN (_col11 is null) THEN (false) WHEN (_col16 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col12 < _col11)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col11 = 0L)) THEN (true) WHEN (_col11 is null) THEN (true) WHEN (_col16 is not null) THEN (false) WHEN (_col1 is null) THEN (null) WHEN ((_col12 < _col11)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 13 Data size: 8187 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) @@ -2971,22 +3002,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: string), _col0 (type: int), _col2 (type: int) - outputColumnNames: _col0, _col1, _col2 + expressions: _col1 (type: string), _col0 (type: int), _col2 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 13 Data size: 1729 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col1 (type: int), _col0 (type: string), _col2 (type: int) + sort order: +++ + Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col2 (type: int) Statistics: Num rows: 13 Data size: 1729 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 13 Data size: 1729 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int), _col0 (type: string), _col2 (type: int) - sort order: +++ - Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col2 (type: int) - Statistics: Num rows: 13 Data size: 1729 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: boolean) + value expressions: _col3 (type: boolean) Stage: Stage-0 Fetch Operator @@ -3062,6 +3086,9 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Filter Operator + predicate: (p_brand is not null and p_type is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: (UDFToDouble(p_type) + 2.0D) (type: double), p_brand (type: string) outputColumnNames: _col0, _col1 @@ -3107,7 +3134,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col4, _col5, _col8 Statistics: Num rows: 26 Data size: 6262 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col4 = 0L)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col1 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 13 Data size: 3133 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) @@ -3143,19 +3170,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 13 Data size: 1300 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: double), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: double), _col1 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: double), _col1 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: string) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator @@ -3184,12 +3208,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: + Reducer 10 <- Map 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Reducer 5 (SIMPLE_EDGE) - Reducer 9 <- Map 8 (SIMPLE_EDGE) + Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 10 (ONE_TO_ONE_EDGE) + Reducer 8 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3214,39 +3239,76 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: p_size is not null (type: boolean) Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: (_col1 + 1) (type: int) - sort order: + - Map-reduce partition columns: (_col1 + 1) (type: int) + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + Reduce Output Operator + key expressions: (_col1 + 1) (type: int) + sort order: + + Map-reduce partition columns: (_col1 + 1) (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Filter Operator + predicate: (p_size is not null and p_type is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: p_type (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (_col1 + 1) (type: int) + sort order: + + Map-reduce partition columns: (_col1 + 1) (type: int) + Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs - Map 8 + Map 9 Map Operator Tree: TableScan alias: part Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (p_size + 1) (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: p_size is not null (type: boolean) Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash + Select Operator + expressions: (p_size + 1) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs + Reducer 10 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -3276,7 +3338,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col4, _col5, _col8 Statistics: Num rows: 26 Data size: 6046 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col4 = 0L)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col1 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 13 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) @@ -3312,16 +3374,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 8 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Group By Operator - keys: _col0 (type: string), _col2 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reducer 6 Execution mode: vectorized, llap Reduce Operator Tree: @@ -3338,6 +3390,27 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 (_col1 + 1) (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 16 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col2 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 8 Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator @@ -3345,32 +3418,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: string), _col1 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 8 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 8 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) - Reducer 9 - Execution mode: vectorized, llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator @@ -3744,7 +3801,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 25750 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Filter Operator - predicate: key is not null (type: boolean) + predicate: (concat('v', value) is not null and key is not null) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: concat('v', value) (type: string), key (type: string) @@ -3815,7 +3872,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col3, _col4, _col7 Statistics: Num rows: 500 Data size: 97100 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 250 Data size: 48560 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) @@ -3893,22 +3950,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 67750 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: string), _col0 (type: string) - outputColumnNames: _col0, _col1 + expressions: _col1 (type: string), _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: string) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 250 Data size: 68750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Reducer 9 Execution mode: vectorized, llap Reduce Operator Tree: @@ -5167,6 +5217,9 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Filter Operator + predicate: (p_brand is not null and p_type is not null) (type: boolean) + Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_brand (type: string), p_type (type: string) mode: hash @@ -5184,16 +5237,19 @@ STAGE PLANS: TableScan alias: e Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: p_size is not null (type: boolean) + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: p_size (type: int) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 10 @@ -5239,7 +5295,7 @@ STAGE PLANS: outputColumnNames: _col1, _col3, _col4, _col7 Statistics: Num rows: 26 Data size: 492 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN ((_col1 + 100) is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN ((_col1 + 100) is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 13 Data size: 248 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 13 Data size: 248 Basic stats: COMPLETE Column stats: COMPLETE @@ -5566,16 +5622,19 @@ STAGE PLANS: TableScan alias: t2 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: c1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) + Filter Operator + predicate: c1 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: no inputs Map 8 @@ -5583,16 +5642,19 @@ STAGE PLANS: TableScan alias: t1 Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: c2 (type: char(100)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: char(100)) - sort order: + - Map-reduce partition columns: _col0 (type: char(100)) - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: c2 is not null (type: boolean) + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: c2 (type: char(100)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: char(100)) + sort order: + + Map-reduce partition columns: _col0 (type: char(100)) + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -5624,7 +5686,7 @@ STAGE PLANS: outputColumnNames: _col0, _col3, _col4, _col7 Statistics: Num rows: 4 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 2 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) @@ -5647,7 +5709,7 @@ STAGE PLANS: 0 UDFToDouble(_col0) (type: double) 1 UDFToDouble(_col0) (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col0) keys: _col1 (type: char(100)) @@ -5693,19 +5755,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: char(100)), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: int), _col1 (type: char(100)), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: char(100)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: char(100)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Reducer 9 Execution mode: vectorized, llap Reduce Operator Tree: @@ -5713,12 +5772,12 @@ STAGE PLANS: keys: KEY._col0 (type: char(100)) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: UDFToDouble(_col0) (type: double) sort order: + Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: char(100)) Stage: Stage-0 @@ -5847,6 +5906,9 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Filter Operator + predicate: (a is not null and b is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: b (type: int), a (type: int) mode: hash @@ -5888,15 +5950,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col3, _col4, _col7 Statistics: Num rows: 3 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) - Statistics: Num rows: 2 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE + predicate: CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END (type: boolean) + Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -5925,22 +5987,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), _col0 (type: int) - outputColumnNames: _col0, _col1 + expressions: _col1 (type: int), _col0 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Reduce Output Operator + key expressions: _col1 (type: int), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: int), _col0 (type: int) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: int), _col0 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator @@ -6067,6 +6122,9 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Filter Operator + predicate: (i is not null and j is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: i (type: int), j (type: int) mode: hash @@ -6084,16 +6142,19 @@ STAGE PLANS: TableScan alias: fixob Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: j is not null (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -6110,7 +6171,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col3, _col4, _col6 Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col6 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col6 is not null) THEN (false) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int) @@ -6289,7 +6350,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Filter Operator - predicate: j is not null (type: boolean) + predicate: (i is not null and j is not null) (type: boolean) Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: i (type: int), j (type: int) @@ -6308,16 +6369,19 @@ STAGE PLANS: TableScan alias: t Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: j (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: j is not null (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: j (type: int) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -6334,7 +6398,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col3, _col4, _col6 Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col6 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col6 is not null) THEN (false) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) @@ -6478,7 +6542,7 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Filter Operator - predicate: j is not null (type: boolean) + predicate: (i is not null and j is not null) (type: boolean) Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: i (type: int), j (type: int) @@ -6521,15 +6585,15 @@ STAGE PLANS: outputColumnNames: _col0, _col3, _col4, _col7 Statistics: Num rows: 3 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) - Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + predicate: CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END (type: boolean) + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -6557,19 +6621,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: int), _col1 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator @@ -6891,7 +6952,7 @@ PREHOOK: query: drop table t1 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table t1 POSTHOOK: type: DROPTABLE -Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select * from src b where b.key not in @@ -7013,7 +7074,7 @@ STAGE PLANS: residual filter predicates: {(_col1 > _col6)} Statistics: Num rows: 500 Data size: 104497 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 250 Data size: 52304 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string) @@ -7077,19 +7138,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string), _col2 (type: boolean) + value expressions: _col1 (type: string), _col2 (type: boolean) Reducer 8 Execution mode: vectorized, llap Reduce Operator Tree: @@ -7109,7 +7167,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select * from src b where b.key not in diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out index 5891519960..67c4323cfc 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out @@ -1326,16 +1326,19 @@ STAGE PLANS: TableScan alias: e Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: p_name is not null (type: boolean) Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) Execution mode: vectorized, llap LLAP IO: no inputs Map 4 @@ -1368,9 +1371,9 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string) - 1 _col2 (type: string) - outputColumnNames: _col1, _col2, _col3 - residual filter predicates: {((_col1 + 100) < CASE WHEN (_col3 is null) THEN (null) ELSE (_col2) END)} + 1 _col1 (type: string) + outputColumnNames: _col1, _col2 + residual filter predicates: {((_col1 + 100) < _col2)} Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE @@ -1408,15 +1411,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), true (type: boolean), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col2 (type: string) + key expressions: _col1 (type: string) sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: boolean) + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) Stage: Stage-0 Fetch Operator @@ -1456,15 +1459,18 @@ STAGE PLANS: TableScan alias: e Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_name (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: p_name is not null (type: boolean) Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: p_name (type: string) + outputColumnNames: _col0 Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map 4 @@ -1582,16 +1588,19 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Filter Operator + predicate: p_type is not null (type: boolean) Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: string) - sort order: + - Map-reduce partition columns: _col4 (type: string) + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Reduce Output Operator + key expressions: _col4 (type: string) + sort order: + + Map-reduce partition columns: _col4 (type: string) + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map 3 @@ -1624,9 +1633,9 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col4 (type: string) - 1 _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - residual filter predicates: {(UDFToDouble(_col5) > CASE WHEN (_col10 is null) THEN (null) ELSE (_col9) END)} + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + residual filter predicates: {(UDFToDouble(_col5) > _col9)} Statistics: Num rows: 9 Data size: 5690 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) @@ -1649,15 +1658,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (_col1 / _col2) (type: double), true (type: boolean), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: (_col1 / _col2) (type: double), _col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col2 (type: string) + key expressions: _col1 (type: string) sort order: + - Map-reduce partition columns: _col2 (type: string) + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: double), _col1 (type: boolean) + value expressions: _col0 (type: double) Stage: Stage-0 Fetch Operator @@ -1870,7 +1879,7 @@ POSTHOOK: Input: default@part_null 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where p_size >= (select min(p_size) from part_null where part_null.p_type = part.p_type) AND p_retailprice <= (select max(p_retailprice) from part_null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_size >= (select min(p_size) from part_null where part_null.p_type = part.p_type) AND p_retailprice <= (select max(p_retailprice) from part_null) @@ -1895,16 +1904,19 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Filter Operator + predicate: p_type is not null (type: boolean) Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: string) - sort order: + - Map-reduce partition columns: _col4 (type: string) + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Reduce Output Operator + key expressions: _col4 (type: string) + sort order: + + Map-reduce partition columns: _col4 (type: string) + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map 4 @@ -1957,9 +1969,9 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col4 (type: string) - 1 _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - residual filter predicates: {(_col5 >= CASE WHEN (_col10 is null) THEN (null) ELSE (_col9) END)} + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + residual filter predicates: {(_col5 >= _col9)} Statistics: Num rows: 9 Data size: 5690 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) @@ -2002,15 +2014,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: int), true (type: boolean), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: _col1 (type: int), _col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col2 (type: string) + key expressions: _col1 (type: string) sort order: + - Map-reduce partition columns: _col2 (type: string) + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: boolean) + value expressions: _col0 (type: int) Reducer 7 Execution mode: vectorized, llap Reduce Operator Tree: @@ -2030,7 +2042,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from part where p_size >= (select min(p_size) from part_null where part_null.p_type = part.p_type) AND p_retailprice <= (select max(p_retailprice) from part_null) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3112,16 +3124,19 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 + Filter Operator + predicate: p_size is not null (type: boolean) Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map 3 @@ -3172,9 +3187,9 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col2 (type: int) - 1 _col2 (type: int) - outputColumnNames: _col0, _col1, _col3, _col4 - residual filter predicates: {(_col1 like CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END)} + 1 _col1 (type: int) + outputColumnNames: _col0, _col1, _col3 + residual filter predicates: {(_col1 like _col3)} Statistics: Num rows: 8 Data size: 2472 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) @@ -3220,15 +3235,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 14 Data size: 2632 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: string), true (type: boolean), _col0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 14 Data size: 2688 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: string), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 2632 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col1 (type: int) sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 14 Data size: 2688 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: boolean) + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 14 Data size: 2632 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator @@ -3480,7 +3495,7 @@ POSTHOOK: Input: default@part_null 85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull 86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl -Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select * from part_null where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null pp where part_null.p_type = pp.p_type) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part_null where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null pp where part_null.p_type = pp.p_type) @@ -3507,14 +3522,17 @@ STAGE PLANS: TableScan alias: part_null Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Filter Operator + predicate: p_type is not null (type: boolean) Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1120 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map 5 @@ -3598,17 +3616,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col12 Statistics: Num rows: 14 Data size: 1787 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col9 = 0L)) THEN (false) WHEN (_col12 is not null) THEN (true) WHEN (_col3 is null) THEN (null) WHEN ((_col10 < _col9)) THEN (true) ELSE (false) END) (type: boolean) - Statistics: Num rows: 7 Data size: 893 Basic stats: COMPLETE Column stats: NONE + predicate: ((_col12 is null and _col3 is not null and (_col10 >= _col9)) or (_col9 = 0L)) (type: boolean) + Statistics: Num rows: 9 Data size: 1148 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 7 Data size: 893 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1148 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col4 (type: string) sort order: + Map-reduce partition columns: _col4 (type: string) - Statistics: Num rows: 7 Data size: 893 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 1148 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Reducer 4 Execution mode: llap @@ -3618,17 +3636,17 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col4 (type: string) - 1 _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col13, _col14 - residual filter predicates: {(not (_col1 like CASE WHEN (_col14 is null) THEN (null) ELSE (_col13) END))} - Statistics: Num rows: 4 Data size: 561 Basic stats: COMPLETE Column stats: NONE + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col13 + residual filter predicates: {(not (_col1 like _col13))} + Statistics: Num rows: 5 Data size: 701 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 4 Data size: 561 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 701 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 4 Data size: 561 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 701 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3673,15 +3691,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col1 (type: string), true (type: boolean), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: _col1 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col2 (type: string) + key expressions: _col1 (type: string) sort order: + - Map-reduce partition columns: _col2 (type: string) + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: boolean) + value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator @@ -3689,7 +3707,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[42][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[43][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select * from part_null where p_brand NOT IN (select p_name from part) AND p_name NOT LIKE (select min(p_name) from part_null pp where part_null.p_type = pp.p_type) PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -3701,6 +3719,7 @@ POSTHOOK: Input: default@part POSTHOOK: Input: default@part_null #### A masked pattern was here #### 90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -3722,7 +3741,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (XPROD_EDGE), Reducer 7 (XPROD_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -3755,15 +3774,15 @@ STAGE PLANS: predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) + value expressions: _col0 (type: int), _col2 (type: int) Execution mode: vectorized, llap LLAP IO: no inputs Map 6 @@ -3772,15 +3791,15 @@ STAGE PLANS: alias: lineitem Statistics: Num rows: 100 Data size: 9600 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((l_linenumber = 1) and (l_shipmode = 'AIR')) (type: boolean) + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: l_orderkey (type: int), l_linenumber (type: int) - outputColumnNames: l_orderkey, l_linenumber - Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE + expressions: l_orderkey (type: int) + outputColumnNames: _col1 + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: min(l_orderkey) - keys: l_linenumber (type: int) + aggregations: min(_col1) + keys: 1 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -3814,13 +3833,11 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col1 (type: int) - outputColumnNames: _col0, _col1, _col3, _col4 - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col4 (type: int) - sort order: + - Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + sort order: + Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int) Reducer 4 Execution mode: llap @@ -3829,10 +3846,10 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col4 (type: int) - 1 _col2 (type: int) - outputColumnNames: _col0, _col1, _col3, _col5, _col6 - residual filter predicates: {(_col1 <> CASE WHEN (_col6 is null) THEN (null) ELSE (_col5) END)} + 0 + 1 + outputColumnNames: _col0, _col1, _col3, _col5 + residual filter predicates: {(_col1 <> _col5)} Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col3 (type: int) @@ -3855,15 +3872,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), true (type: boolean), _col0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: boolean) + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) Stage: Stage-0 Fetch Operator @@ -3871,6 +3886,7 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -3910,6 +3926,7 @@ POSTHOOK: Input: default@lineitem 85951 5952 88035 5560 88362 871 +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: explain select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -3931,7 +3948,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (XPROD_EDGE), Reducer 7 (XPROD_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -3964,15 +3981,15 @@ STAGE PLANS: predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean) Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 + expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 14 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int) + value expressions: _col0 (type: int), _col2 (type: int) Execution mode: vectorized, llap LLAP IO: no inputs Map 6 @@ -3981,15 +3998,15 @@ STAGE PLANS: alias: lineitem Statistics: Num rows: 100 Data size: 9600 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((l_linenumber = 1) and (l_shipmode = 'AIR')) (type: boolean) + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: l_orderkey (type: int), l_linenumber (type: int) - outputColumnNames: l_orderkey, l_linenumber - Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE + expressions: l_orderkey (type: int) + outputColumnNames: _col1 + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: min(l_orderkey) - keys: l_linenumber (type: int) + aggregations: min(_col1) + keys: 1 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -4023,13 +4040,11 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col1 (type: int) - outputColumnNames: _col0, _col1, _col3, _col4 - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col4 (type: int) - sort order: + - Map-reduce partition columns: _col4 (type: int) - Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + sort order: + Statistics: Num rows: 7 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int) Reducer 4 Execution mode: llap @@ -4038,10 +4053,10 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col4 (type: int) - 1 _col2 (type: int) - outputColumnNames: _col0, _col1, _col3, _col5, _col6 - residual filter predicates: {(_col1 <> CASE WHEN (_col6 is null) THEN (null) ELSE (_col5) END)} + 0 + 1 + outputColumnNames: _col0, _col1, _col3, _col5 + residual filter predicates: {(_col1 <> _col5)} Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col3 (type: int) @@ -4064,15 +4079,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), true (type: boolean), _col0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: boolean) + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) Stage: Stage-0 Fetch Operator @@ -4080,6 +4093,7 @@ STAGE PLANS: Processor Tree: ListSink +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select p.p_partkey, li.l_suppkey from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey where li.l_linenumber = 1 and @@ -4208,9 +4222,9 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - 2 _col1 (type: int) - outputColumnNames: _col1, _col2, _col4 - residual filter predicates: {(_col1 > _col4)} + 2 _col0 (type: int) + outputColumnNames: _col1, _col2, _col5 + residual filter predicates: {(_col1 > _col5)} Statistics: Num rows: 4 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: double) @@ -4250,15 +4264,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 50 Data size: 1000 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: (_col1 / _col2) (type: double), _col0 (type: int) + expressions: _col0 (type: int), (_col1 / _col2) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 50 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: int) + key expressions: _col0 (type: int) sort order: + - Map-reduce partition columns: _col1 (type: int) + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 50 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: double) + value expressions: _col1 (type: double) Stage: Stage-0 Fetch Operator @@ -4321,7 +4335,7 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: p_type is not null (type: boolean) + predicate: (p_name is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_name (type: string), p_brand (type: string), p_type (type: string) @@ -4383,27 +4397,24 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col2 (type: string) - 1 _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - residual filter predicates: {(not (_col1 like CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END))} + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + residual filter predicates: {(not (_col1 like _col3))} Statistics: Num rows: 7 Data size: 3507 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 7 Data size: 1575 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (_col0 is not null and _col1 is not null) (type: boolean) - Statistics: Num rows: 7 Data size: 1575 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 675 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 3 Data size: 675 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 3 Data size: 675 Basic stats: COMPLETE Column stats: COMPLETE Reducer 6 Execution mode: vectorized, llap Reduce Operator Tree: @@ -4414,15 +4425,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 3744 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: string), true (type: boolean), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 13 Data size: 3796 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3744 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col2 (type: string) + key expressions: _col1 (type: string) sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 13 Data size: 3796 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: boolean) + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 13 Data size: 3744 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator @@ -4745,16 +4756,19 @@ STAGE PLANS: TableScan alias: emps Statistics: Num rows: 5 Data size: 3160 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: empno (type: int), name (type: string), deptno (type: int), gender (type: string), city (type: string), empid (type: int), age (type: int), slacker (type: boolean), manager (type: boolean), joinedat (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Filter Operator + predicate: deptno is not null (type: boolean) Statistics: Num rows: 5 Data size: 3160 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - sort order: + - Map-reduce partition columns: _col2 (type: int) + Select Operator + expressions: empno (type: int), name (type: string), deptno (type: int), gender (type: string), city (type: string), empid (type: int), age (type: int), slacker (type: boolean), manager (type: boolean), joinedat (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 5 Data size: 3160 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + Statistics: Num rows: 5 Data size: 3160 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) Execution mode: vectorized, llap LLAP IO: no inputs Map 3 @@ -4787,9 +4801,9 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col2 (type: int) - 1 _col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - residual filter predicates: {(_col1 > CASE WHEN (_col11 is null) THEN (null) ELSE (_col10) END)} + 1 _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + residual filter predicates: {(_col1 > _col10)} Statistics: Num rows: 1 Data size: 695 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) @@ -4812,15 +4826,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: string), true (type: boolean), _col0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: string), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col2 (type: int) + key expressions: _col1 (type: int) sort order: + - Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: boolean) + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) Stage: Stage-0 Fetch Operator @@ -5021,7 +5035,7 @@ POSTHOOK: Input: default@emps 110 John 40 M Vancouver 2 NULL false true 2002-05-03 120 Wilma 20 F NULL 1 5 NULL true 2005-09-07 130 Alice 40 F Vancouver 2 NULL false true 2007-01-01 -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from emps where deptno <> (select sum(deptno) from depts where depts.name = emps.name) and empno > (select count(name) from depts) PREHOOK: type: QUERY POSTHOOK: query: explain select * from emps where deptno <> (select sum(deptno) from depts where depts.name = emps.name) and empno > (select count(name) from depts) @@ -5046,16 +5060,19 @@ STAGE PLANS: TableScan alias: emps Statistics: Num rows: 5 Data size: 3160 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: empno (type: int), name (type: string), deptno (type: int), gender (type: string), city (type: string), empid (type: int), age (type: int), slacker (type: boolean), manager (type: boolean), joinedat (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Filter Operator + predicate: name is not null (type: boolean) Statistics: Num rows: 5 Data size: 3160 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Select Operator + expressions: empno (type: int), name (type: string), deptno (type: int), gender (type: string), city (type: string), empid (type: int), age (type: int), slacker (type: boolean), manager (type: boolean), joinedat (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 5 Data size: 3160 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 5 Data size: 3160 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) Execution mode: vectorized, llap LLAP IO: no inputs Map 4 @@ -5108,9 +5125,9 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col1 (type: string) - 1 _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - residual filter predicates: {(UDFToLong(_col2) <> CASE WHEN (_col11 is null) THEN (null) ELSE (_col10) END)} + 1 _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + residual filter predicates: {(UDFToLong(_col2) <> _col10)} Statistics: Num rows: 5 Data size: 3476 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: boolean), _col8 (type: boolean), _col9 (type: date) @@ -5153,15 +5170,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: bigint), true (type: boolean), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE + expressions: _col1 (type: bigint), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col2 (type: string) + key expressions: _col1 (type: string) sort order: + - Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: boolean) + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1 Data size: 101 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) Reducer 7 Execution mode: vectorized, llap Reduce Operator Tree: @@ -5726,16 +5743,19 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Filter Operator + predicate: p_type is not null (type: boolean) Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: string) - sort order: + - Map-reduce partition columns: _col4 (type: string) + Select Operator + expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + Reduce Output Operator + key expressions: _col4 (type: string) + sort order: + + Map-reduce partition columns: _col4 (type: string) + Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Map 3 @@ -6154,7 +6174,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: CASE WHEN (_col3 is null) THEN ((0 = 0)) ELSE ((_col2 = 0)) END (type: boolean) + predicate: CASE WHEN (_col3 is null) THEN (true) ELSE ((_col2 = 0L)) END (type: boolean) Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col1 (type: int) @@ -6243,7 +6263,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@t #### A masked pattern was here #### 1 1 -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from t where 0 = (select avg(tt.i) from t tt where tt.j <> t.i) PREHOOK: type: QUERY POSTHOOK: query: explain select * from t where 0 = (select avg(tt.i) from t tt where tt.j <> t.i) @@ -6268,16 +6288,23 @@ STAGE PLANS: TableScan alias: t Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: i is not null (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) Select Operator expressions: i (type: int), j (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) Reduce Output Operator sort order: Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE @@ -6289,16 +6316,19 @@ STAGE PLANS: TableScan alias: t Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: i (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Filter Operator + predicate: i is not null (type: boolean) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: i (type: int) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 @@ -6387,7 +6417,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[31][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from t where 0 = (select avg(tt.i) from t tt where tt.j <> t.i) PREHOOK: type: QUERY PREHOOK: Input: default@t @@ -6472,7 +6502,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: CASE WHEN (_col3 is null) THEN ((0 = 0)) ELSE ((_col2 = 0)) END (type: boolean) + predicate: CASE WHEN (_col3 is null) THEN (true) ELSE ((_col2 = 0L)) END (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int) @@ -6545,16 +6575,19 @@ STAGE PLANS: TableScan alias: t Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 + Filter Operator + predicate: i is not null (type: boolean) Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Select Operator + expressions: i (type: int), j (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) Execution mode: vectorized, llap LLAP IO: no inputs Map 3 diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out b/ql/src/test/results/clientpositive/llap/subquery_select.q.out index fec99240f3..408ac92828 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out @@ -322,23 +322,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int), _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator @@ -673,17 +669,17 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col3 Statistics: Num rows: 26 Data size: 2928 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (sq_count_check(_col3, true) > 0) (type: boolean) + predicate: (sq_count_check(CASE WHEN (_col3 is null) THEN (0) ELSE (_col3) END, true) > 0) (type: boolean) Statistics: Num rows: 8 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 8 Data size: 904 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) Reducer 3 Execution mode: llap @@ -694,14 +690,14 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col5, _col6 + outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 8 Data size: 944 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 944 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col5 (type: bigint), _col6 (type: bigint) + value expressions: _col3 (type: bigint), _col4 (type: bigint) Reducer 4 Execution mode: llap Reduce Operator Tree: @@ -711,10 +707,10 @@ STAGE PLANS: keys: 0 _col0 (type: string), _col1 (type: int) 1 _col1 (type: string), _col0 (type: int) - outputColumnNames: _col1, _col5, _col6, _col9 + outputColumnNames: _col1, _col3, _col4, _col7 Statistics: Num rows: 8 Data size: 132 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col1 (type: int), CASE WHEN ((_col5 = 0L)) THEN (true) WHEN (_col5 is null) THEN (true) WHEN (_col9 is not null) THEN (false) WHEN (_col1 is null) THEN (null) WHEN ((_col6 < _col5)) THEN (null) ELSE (true) END (type: boolean) + expressions: _col1 (type: int), CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col1 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (null) ELSE (true) END (type: boolean) outputColumnNames: _col0, _col1 Statistics: Num rows: 8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator @@ -754,23 +750,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int), _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Reducer 8 Execution mode: vectorized, llap Reduce Operator Tree: @@ -3064,7 +3056,7 @@ STANDARD ANODIZED TIN true STANDARD BURNISHED TIN true STANDARD PLATED TIN true STANDARD POLISHED STEEL true -Warning: Shuffle Join MERGEJOIN[63][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[62][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: EXPLAIN SELECT p_size, p_size IN ( SELECT MAX(p_size) FROM part p where p.p_type = part.p_type) AND p_name IN (SELECT min(p_name) from part) @@ -3286,23 +3278,19 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int), _col0 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: int) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator @@ -3310,7 +3298,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[63][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[62][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product PREHOOK: query: SELECT p_size, p_size IN ( SELECT MAX(p_size) FROM part p where p.p_type = part.p_type) AND p_name IN (SELECT min(p_name) from part) @@ -4825,7 +4813,7 @@ STAGE PLANS: alias: p Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (p_size is not null and p_type is not null) (type: boolean) + predicate: (p_partkey is not null and p_size is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_partkey (type: int), p_type (type: string), p_size (type: int) @@ -5119,7 +5107,7 @@ STAGE PLANS: alias: p Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (p_size is not null and p_type is not null) (type: boolean) + predicate: (p_partkey is not null and p_size is not null and p_type is not null) (type: boolean) Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: p_partkey (type: int), p_type (type: string), p_size (type: int) diff --git a/ql/src/test/results/clientpositive/llap/subquery_views.q.out b/ql/src/test/results/clientpositive/llap/subquery_views.q.out index 2c8530933c..4609668b46 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -132,33 +132,33 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((value > 'val_11') and key is not null) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((key < '11') and (value > 'val_11')) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(key) keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5238 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5238 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Filter Operator - predicate: ((value > 'val_11') and key is not null) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((key < '11') and (value > 'val_11')) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (key < '11') (type: boolean) Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE @@ -172,19 +172,19 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((value > 'val_11') and key is not null) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + predicate: ((key < '11') and (value > 'val_11')) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(key) keys: key (type: string), value (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5238 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5238 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs @@ -195,16 +195,19 @@ STAGE PLANS: properties: insideView TRUE Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (key < '11') (type: boolean) + Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Reducer 10 @@ -214,12 +217,12 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -232,19 +235,19 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) 2 _col3 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1, _col4, _col5, _col8 - Statistics: Num rows: 87 Data size: 17226 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 5148 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) - Statistics: Num rows: 43 Data size: 8514 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 2574 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 43 Data size: 7654 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 2314 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 43 Data size: 7654 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 2314 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) Reducer 3 Execution mode: llap @@ -256,10 +259,10 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 43 Data size: 7654 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 2314 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 43 Data size: 7654 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 2314 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -272,12 +275,12 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5238 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5238 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Reducer 5 Execution mode: vectorized, llap @@ -286,16 +289,16 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4806 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4914 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 4914 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string), _col2 (type: boolean) Reducer 6 Execution mode: llap @@ -307,18 +310,18 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 3822 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col3 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col3 (type: string), _col1 (type: string) - Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 3822 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) Reduce Output Operator key expressions: _col3 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col3 (type: string), _col1 (type: string) - Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 21 Data size: 3822 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) Reducer 7 Execution mode: llap @@ -332,24 +335,24 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) 2 _col3 (type: string), _col1 (type: string) outputColumnNames: _col0, _col4, _col5, _col8 - Statistics: Num rows: 87 Data size: 9309 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 26 Data size: 2782 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean) - Statistics: Num rows: 43 Data size: 4601 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1391 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 43 Data size: 3741 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 13 Data size: 1131 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 8 Data size: 696 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 8 Data size: 696 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE Reducer 8 Execution mode: vectorized, llap Reduce Operator Tree: @@ -358,12 +361,12 @@ STAGE PLANS: keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5238 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 27 Data size: 5238 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) Stage: Stage-0 diff --git a/ql/src/test/results/clientpositive/perf/tez/query1.q.out b/ql/src/test/results/clientpositive/perf/tez/query1.q.out index eb1f83c9a5..ea6e0dab10 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query1.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query1.q.out @@ -73,9 +73,9 @@ Stage-0 Select Operator [SEL_49] (rows=32266667 width=860) Output:["_col0"] Filter Operator [FIL_48] (rows=32266667 width=860) - predicate:(_col2 > CASE WHEN (_col8 is null) THEN (null) ELSE (_col7) END) + predicate:(_col2 > _col7) Merge Join Operator [MERGEJOIN_81] (rows=96800003 width=860) - Conds:RS_45._col1=RS_105._col2(Inner),Output:["_col2","_col6","_col7","_col8"] + Conds:RS_45._col1=RS_105._col1(Inner),Output:["_col2","_col6","_col7"] <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_45] PartitionCols:_col1 @@ -138,9 +138,9 @@ Stage-0 default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] <-Reducer 9 [SIMPLE_EDGE] vectorized SHUFFLE [RS_105] - PartitionCols:_col2 + PartitionCols:_col1 Select Operator [SEL_104] (rows=15837566 width=77) - Output:["_col0","_col1","_col2"] + Output:["_col0","_col1"] Group By Operator [GBY_103] (rows=15837566 width=77) Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)","count(_col2)"],keys:_col1 Select Operator [SEL_102] (rows=31675133 width=77) diff --git a/ql/src/test/results/clientpositive/perf/tez/query30.q.out b/ql/src/test/results/clientpositive/perf/tez/query30.q.out index 7e5dc5a3e3..4d0e2d4cc4 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query30.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query30.q.out @@ -117,14 +117,14 @@ Stage-0 Select Operator [SEL_55] (rows=8066666 width=1014) Output:["_col0","_col2"] Filter Operator [FIL_54] (rows=8066666 width=1014) - predicate:(_col2 > CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END) + predicate:(_col2 > _col3) Merge Join Operator [MERGEJOIN_106] (rows=24200000 width=1014) - Conds:RS_132._col1=RS_137._col2(Inner),Output:["_col0","_col2","_col3","_col4"] + Conds:RS_132._col1=RS_137._col1(Inner),Output:["_col0","_col2","_col3"] <-Reducer 10 [SIMPLE_EDGE] vectorized SHUFFLE [RS_137] - PartitionCols:_col2 + PartitionCols:_col1 Select Operator [SEL_136] (rows=11000000 width=1014) - Output:["_col0","_col1","_col2"] + Output:["_col0","_col1"] Group By Operator [GBY_135] (rows=11000000 width=1014) Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)","count(_col2)"],keys:_col0 Select Operator [SEL_134] (rows=22000000 width=1014) @@ -189,7 +189,7 @@ Stage-0 Select Operator [SEL_115] (rows=40000000 width=1014) Output:["_col0","_col1"] Filter Operator [FIL_112] (rows=40000000 width=1014) - predicate:ca_address_sk is not null + predicate:(ca_address_sk is not null and ca_state is not null) Please refer to the previous TableScan [TS_3] <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_18] diff --git a/ql/src/test/results/clientpositive/perf/tez/query6.q.out b/ql/src/test/results/clientpositive/perf/tez/query6.q.out index 0e71d5dc36..bafd1757de 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query6.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query6.q.out @@ -95,23 +95,23 @@ Stage-0 Select Operator [SEL_54] (rows=169400 width=1436) Output:["_col0"] Filter Operator [FIL_53] (rows=169400 width=1436) - predicate:(_col5 > (1.2 * CASE WHEN (_col1 is null) THEN (null) ELSE (_col0) END)) + predicate:(_col4 > (1.2 * CAST( _col0 AS decimal(16,6)))) Merge Join Operator [MERGEJOIN_112] (rows=508200 width=1436) - Conds:RS_50._col2=RS_153._col2(Inner),Output:["_col0","_col1","_col4","_col5"] + Conds:RS_50._col1=RS_153._col2(Inner),Output:["_col0","_col3","_col4"] <-Map 20 [SIMPLE_EDGE] vectorized SHUFFLE [RS_153] PartitionCols:_col2 Select Operator [SEL_152] (rows=462000 width=1436) Output:["_col0","_col1","_col2"] Filter Operator [FIL_151] (rows=462000 width=1436) - predicate:i_item_sk is not null + predicate:(i_category is not null and i_item_sk is not null) TableScan [TS_44] (rows=462000 width=1436) default@item,i,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_category"] <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_50] - PartitionCols:_col2 + PartitionCols:_col1 Merge Join Operator [MERGEJOIN_111] (rows=231000 width=1445) - Conds:(Inner),Output:["_col0","_col1","_col2"] + Conds:(Inner),Output:["_col0","_col1"] <-Reducer 11 [CUSTOM_SIMPLE_EDGE] vectorized PARTITION_ONLY_SHUFFLE [RS_144] Select Operator [SEL_143] (rows=1 width=8) @@ -140,7 +140,7 @@ Stage-0 <-Reducer 19 [CUSTOM_SIMPLE_EDGE] vectorized PARTITION_ONLY_SHUFFLE [RS_150] Select Operator [SEL_149] (rows=231000 width=1436) - Output:["_col0","_col1","_col2"] + Output:["_col0","_col1"] Group By Operator [GBY_148] (rows=231000 width=1436) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 <-Map 18 [SIMPLE_EDGE] vectorized diff --git a/ql/src/test/results/clientpositive/perf/tez/query81.q.out b/ql/src/test/results/clientpositive/perf/tez/query81.q.out index 1a6910eb9f..376500e52b 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query81.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query81.q.out @@ -77,12 +77,12 @@ Stage-0 limit:-1 Stage-1 Reducer 4 vectorized - File Output Operator [FS_142] - Select Operator [SEL_141] (rows=100 width=860) + File Output Operator [FS_140] + Select Operator [SEL_139] (rows=100 width=860) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15"] - Limit [LIM_140] (rows=100 width=860) + Limit [LIM_138] (rows=100 width=860) Number of rows:100 - Select Operator [SEL_139] (rows=96800003 width=860) + Select Operator [SEL_137] (rows=96800003 width=860) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_63] @@ -96,19 +96,19 @@ Stage-0 Select Operator [SEL_55] (rows=8066666 width=1014) Output:["_col0","_col2"] Filter Operator [FIL_54] (rows=8066666 width=1014) - predicate:(_col2 > CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END) + predicate:(_col2 > _col3) Merge Join Operator [MERGEJOIN_107] (rows=24200000 width=1014) - Conds:RS_133._col1=RS_138._col2(Inner),Output:["_col0","_col2","_col3","_col4"] + Conds:RS_131._col1=RS_136._col1(Inner),Output:["_col0","_col2","_col3"] <-Reducer 13 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_138] - PartitionCols:_col2 - Select Operator [SEL_137] (rows=11000000 width=1014) - Output:["_col0","_col1","_col2"] - Group By Operator [GBY_136] (rows=11000000 width=1014) + SHUFFLE [RS_136] + PartitionCols:_col1 + Select Operator [SEL_135] (rows=11000000 width=1014) + Output:["_col0","_col1"] + Group By Operator [GBY_134] (rows=11000000 width=1014) Output:["_col0","_col1","_col2"],aggregations:["sum(_col2)","count(_col2)"],keys:_col0 - Select Operator [SEL_135] (rows=22000000 width=1014) + Select Operator [SEL_133] (rows=22000000 width=1014) Output:["_col0","_col2"] - Group By Operator [GBY_134] (rows=22000000 width=1014) + Group By Operator [GBY_132] (rows=22000000 width=1014) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_43] @@ -116,13 +116,13 @@ Stage-0 Group By Operator [GBY_42] (rows=44000000 width=1014) Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col7, _col1 Merge Join Operator [MERGEJOIN_106] (rows=44000000 width=1014) - Conds:RS_38._col2=RS_130._col0(Inner),Output:["_col1","_col3","_col7"] + Conds:RS_38._col2=RS_128._col0(Inner),Output:["_col1","_col3","_col7"] <-Map 15 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_130] + SHUFFLE [RS_128] PartitionCols:_col0 - Select Operator [SEL_128] (rows=40000000 width=1014) + Select Operator [SEL_126] (rows=40000000 width=1014) Output:["_col0","_col1"] - Filter Operator [FIL_126] (rows=40000000 width=1014) + Filter Operator [FIL_125] (rows=40000000 width=1014) predicate:(ca_address_sk is not null and ca_state is not null) TableScan [TS_12] (rows=40000000 width=1014) default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] @@ -150,11 +150,11 @@ Stage-0 TableScan [TS_6] (rows=28798881 width=106) default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_returned_date_sk","cr_returning_customer_sk","cr_returning_addr_sk","cr_return_amt_inc_tax"] <-Reducer 9 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_133] + SHUFFLE [RS_131] PartitionCols:_col1 - Select Operator [SEL_132] (rows=22000000 width=1014) + Select Operator [SEL_130] (rows=22000000 width=1014) Output:["_col0","_col1","_col2"] - Group By Operator [GBY_131] (rows=22000000 width=1014) + Group By Operator [GBY_129] (rows=22000000 width=1014) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1 <-Reducer 8 [SIMPLE_EDGE] SHUFFLE [RS_23] @@ -162,15 +162,11 @@ Stage-0 Group By Operator [GBY_22] (rows=44000000 width=1014) Output:["_col0","_col1","_col2"],aggregations:["sum(_col3)"],keys:_col7, _col1 Merge Join Operator [MERGEJOIN_104] (rows=44000000 width=1014) - Conds:RS_18._col2=RS_129._col0(Inner),Output:["_col1","_col3","_col7"] + Conds:RS_18._col2=RS_127._col0(Inner),Output:["_col1","_col3","_col7"] <-Map 15 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_129] + SHUFFLE [RS_127] PartitionCols:_col0 - Select Operator [SEL_127] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_125] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_126] <-Reducer 7 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col2 diff --git a/ql/src/test/results/clientpositive/subquery_notexists.q.out b/ql/src/test/results/clientpositive/subquery_notexists.q.out index 8b786332e8..01f7470f76 100644 --- a/ql/src/test/results/clientpositive/subquery_notexists.q.out +++ b/ql/src/test/results/clientpositive/subquery_notexists.q.out @@ -647,8 +647,8 @@ POSTHOOK: Input: default@src #### A masked pattern was here #### 98 val_98 98 val_98 -Warning: Shuffle Join JOIN[28][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -Warning: Shuffle Join JOIN[12][tables = [$hdt$_3, $hdt$_4]] in Stage 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[14][tables = [$hdt$_3, $hdt$_4]] in Stage 'Stage-6:MAPRED' is a cross product PREHOOK: query: explain SELECT p1.p_name FROM part p1 LEFT JOIN (select p_type as p_col from part ) p2 WHERE NOT EXISTS (select pp1.p_type as p_col from part pp1 where pp1.p_partkey = p2.p_col) PREHOOK: type: QUERY @@ -761,18 +761,21 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_type (type: string) - outputColumnNames: _col0 + Filter Operator + predicate: p_type is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: + Select Operator + expressions: p_type (type: string) + outputColumnNames: _col0 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + Reduce Output Operator + sort order: + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) Reduce Operator Tree: Join Operator condition map: - Left Outer Join 0 to 1 + Inner Join 0 to 1 keys: 0 1 @@ -819,15 +822,18 @@ STAGE PLANS: TableScan alias: pp1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_partkey (type: int) - outputColumnNames: _col0 + Filter Operator + predicate: p_partkey is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Select Operator + expressions: p_partkey (type: int) + outputColumnNames: _col0 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: UDFToDouble(_col0) (type: double) @@ -889,8 +895,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[28][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -Warning: Shuffle Join JOIN[12][tables = [$hdt$_3, $hdt$_4]] in Stage 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[14][tables = [$hdt$_3, $hdt$_4]] in Stage 'Stage-6:MAPRED' is a cross product PREHOOK: query: SELECT p1.p_name FROM part p1 LEFT JOIN (select p_type as p_col from part ) p2 WHERE NOT EXISTS (select pp1.p_type as p_col from part pp1 where pp1.p_partkey = p2.p_col) PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/subquery_notin_having.q.out b/ql/src/test/results/clientpositive/subquery_notin_having.q.out index ff408b6528..7592820b1d 100644 --- a/ql/src/test/results/clientpositive/subquery_notin_having.q.out +++ b/ql/src/test/results/clientpositive/subquery_notin_having.q.out @@ -335,15 +335,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col3, _col4, _col7 Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col3 = 0L)) THEN (false) WHEN (_col3 is null) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (true) ELSE (false) END) (type: boolean) - Statistics: Num rows: 8 Data size: 1014 Basic stats: COMPLETE Column stats: NONE + predicate: CASE WHEN ((_col3 = 0L)) THEN (true) WHEN (_col3 is null) THEN (true) WHEN (_col7 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col4 < _col3)) THEN (false) ELSE (true) END (type: boolean) + Statistics: Num rows: 7 Data size: 888 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 1014 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 888 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 1014 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 888 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -430,9 +430,8 @@ STAGE PLANS: TableScan alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: p_mfgr (type: string), p_retailprice (type: double) - outputColumnNames: p_mfgr, p_retailprice + Filter Operator + predicate: p_mfgr is not null (type: boolean) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(p_retailprice), max(p_retailprice) @@ -458,22 +457,15 @@ STAGE PLANS: predicate: (((_col2 - _col1) > 600.0D) and _col1 is not null) (type: boolean) Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: double) - outputColumnNames: _col0, _col1 + expressions: _col0 (type: string), _col1 (type: double), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: double), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Fetch Operator @@ -1117,7 +1109,7 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col5 Statistics: Num rows: 4 Data size: 343 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not CASE WHEN ((_col2 = 0L)) THEN (false) WHEN (_col2 is null) THEN (false) WHEN (_col5 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col3 < _col2)) THEN (true) ELSE (false) END) (type: boolean) + predicate: CASE WHEN ((_col2 = 0L)) THEN (true) WHEN (_col2 is null) THEN (true) WHEN (_col5 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col3 < _col2)) THEN (false) ELSE (true) END (type: boolean) Statistics: Num rows: 2 Data size: 171 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) @@ -1242,9 +1234,8 @@ STAGE PLANS: TableScan alias: t1 Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c1 (type: int) - outputColumnNames: c1 + Filter Operator + predicate: c1 is not null (type: boolean) Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: c1 (type: int)