diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java index 336745b..daafeaf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java @@ -34,7 +34,7 @@ Having_clause_without_any_groupby, Hint, Invalid_column_reference, Invalid_decimal, Less_than_equal_greater_than, Multi_insert, Others, Same_name_in_multiple_expressions, Schema_less_table, Select_alias_in_having_clause, Select_transform, Subquery, - Table_sample_clauses, UDTF, Union_type, Unique_join + Table_sample_clauses, UDTF, Union_type, Unique_join, LogicalValues }; private UnsupportedFeature unsupportedFeature; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java new file mode 100644 index 0000000..bbf518d --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.calcite; + +import java.util.HashSet; +import java.util.List; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate; +import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory; +import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter; +import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; + +public class HiveRexExecutorImpl implements RelOptPlanner.Executor { + + private final RelOptCluster cluster; + + public HiveRexExecutorImpl(RelOptCluster cluster) { + this.cluster = cluster; + } + + @Override + public void reduce(RexBuilder rexBuilder, List constExps, List reducedValues) { + RexNodeConverter rexNodeConverter = new RexNodeConverter(cluster); + for (RexNode rexNode : constExps) { + // initialize the converter + ExprNodeConverter converter = new ExprNodeConverter("", null, null, null, + new HashSet(), cluster.getTypeFactory()); + // convert RexNode to ExprNodeGenericFuncDesc + ExprNodeDesc expr = rexNode.accept(converter); + if (expr instanceof ExprNodeGenericFuncDesc) { + // folding the constant + ExprNodeDesc constant = ConstantPropagateProcFactory + .foldExpr((ExprNodeGenericFuncDesc) expr); + if (constant != null) { + try { + // convert constant back to RexNode + reducedValues.add(rexNodeConverter.convert((ExprNodeConstantDesc) constant)); + } catch (Exception e) { + e.printStackTrace(); + reducedValues.add(rexNode); + } + } else { + reducedValues.add(rexNode); + } + } else { + reducedValues.add(rexNode); + } + } + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java index 1f5d919..1428ab1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java @@ -150,8 +150,9 @@ static ASTNode literal(RexLiteral literal, boolean useTypeQualInLiteral) { switch (sqlType) { case BINARY: ByteString bs = (ByteString) literal.getValue(); - val = bs.byteAt(0); - type = HiveParser.BigintLiteral; + val = bs.toString(); + val = "'" + val + "'"; + type = HiveParser.StringLiteral; break; case TINYINT: if (useTypeQualInLiteral) { @@ -217,7 +218,7 @@ static ASTNode literal(RexLiteral literal, boolean useTypeQualInLiteral) { case TIMESTAMP: { val = literal.getValue(); type = HiveParser.TOK_TIMESTAMPLITERAL; - DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); val = df.format(((Calendar) val).getTime()); val = "'" + val + "'"; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java index d026e58..db22c61 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java @@ -37,6 +37,7 @@ import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexFieldAccess; @@ -54,16 +55,19 @@ import org.apache.calcite.util.ImmutableBitSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter.HiveToken; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.ParseDriver; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import com.google.common.collect.Iterables; @@ -310,6 +314,14 @@ private QueryBlockInfo convertSource(RelNode r) throws CalciteSemanticException String sqAlias = nextAlias(); ast = ASTBuilder.subQuery(unionAST, sqAlias); s = new Schema((Union) r, sqAlias); + } else if (r instanceof LogicalValues) { + // We may reach here when constant folding optimization is on. + // The good solution is to scan the dummy table. + // However, it is impossible to set the row limit as 0 in the AST. + // Thus, we skip CBO for this case and let the NullScanOptimizer to handle + // it. + throw new CalciteSemanticException("In ASTConverter, CBO does not support LogicalValues.", + CalciteSemanticException.UnsupportedFeature.LogicalValues); } else { ASTConverter src = new ASTConverter(r, this.derivedTableCount); ASTNode srcAST = src.convert(); @@ -360,7 +372,9 @@ public void visit(RelNode node, int ordinal, RelNode parent) { } else { ASTConverter.this.orderLimit = (Sort) node; } - } + } else if (node instanceof LogicalValues) { + ASTConverter.this.from = node; + } /* * once the source node is reached; stop traversal for this QB */ @@ -532,7 +546,7 @@ public ASTNode visitCall(RexCall call) { SqlOperator op = call.getOperator(); List astNodeLst = new LinkedList(); if (op.kind == SqlKind.CAST) { - HiveToken ht = TypeConverter.hiveToken(call.getType()); + HiveToken ht = TypeConverter.hiveToken(call); ASTBuilder astBldr = ASTBuilder.construct(ht.type, ht.text); if (ht.args != null) { for (String castArg : ht.args) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java index 631a4ca..09c2bdf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; +import org.apache.calcite.avatica.util.ByteString; import org.apache.calcite.avatica.util.TimeUnit; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.rel.RelNode; @@ -112,6 +113,10 @@ private InputCtx(RelDataType calciteInpDataType, ImmutableMap h private final ImmutableList inputCtxs; private final boolean flattenExpr; + public RexNodeConverter(RelOptCluster cluster) { + this(cluster, new ArrayList(), false); + } + public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType, ImmutableMap nameToPosMap, int offset, boolean flattenExpr) { this.cluster = cluster; @@ -321,6 +326,10 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws CalciteSemanticEx coi); RexNode calciteLiteral = null; + if (value == null) { + return cluster.getRexBuilder().makeLiteral(null, + cluster.getTypeFactory().createSqlType(SqlTypeName.NULL), true); + } // TODO: Verify if we need to use ConstantObjectInspector to unwrap data switch (hiveTypeCategory) { case BOOLEAN: @@ -430,6 +439,8 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws CalciteSemanticEx cluster.getTypeFactory().createSqlType(SqlTypeName.NULL), true); break; case BINARY: + calciteLiteral = cluster.getRexBuilder().makeBinaryLiteral(new ByteString((byte[]) value)); + break; case UNKNOWN: default: throw new RuntimeException("UnSupported Literal"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java index 2825f77..eeeedf8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java @@ -28,6 +28,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.sql.type.SqlTypeName; @@ -321,9 +322,9 @@ public static TypeInfo convertPrimitiveType(RelDataType rType) { } /*********************** Convert Calcite Types To Hive Types ***********************/ - public static HiveToken hiveToken(RelDataType calciteType) { + public static HiveToken hiveToken(RexCall call) { HiveToken ht = null; - + RelDataType calciteType = call.getType(); switch (calciteType.getSqlTypeName()) { case CHAR: { ht = new HiveToken(HiveParser.TOK_CHAR, "TOK_CHAR", String.valueOf(calciteType.getPrecision())); @@ -343,6 +344,14 @@ public static HiveToken hiveToken(RelDataType calciteType) { .getPrecision()), String.valueOf(calciteType.getScale())); } break; + case BINARY: { + if (call.getOperands().get(0).getType().getSqlTypeName().getName().equals("BINARY")) { + ht = new HiveToken(HiveParser.Identifier, "unhex"); + } else { + ht = new HiveToken(HiveParser.Identifier, "binary"); + } + } + break; default: ht = calciteToHiveTypeNameMap.get(calciteType.getSqlTypeName().getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index c005b1a..d9218c9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -39,6 +39,7 @@ import org.antlr.runtime.tree.TreeVisitorAction; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptPlanner.Executor; import org.apache.calcite.plan.RelOptQuery; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptSchema; @@ -59,6 +60,8 @@ import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.metadata.CachingRelMetadataProvider; import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataProvider; @@ -73,6 +76,9 @@ import org.apache.calcite.rel.rules.SemiJoinJoinTransposeRule; import org.apache.calcite.rel.rules.SemiJoinProjectTransposeRule; import org.apache.calcite.rel.rules.UnionMergeRule; +import org.apache.calcite.rel.rules.ReduceExpressionsRule.ProjectReduceExpressionsRule; +import org.apache.calcite.rel.rules.ReduceExpressionsRule.FilterReduceExpressionsRule; +import org.apache.calcite.rel.rules.ReduceExpressionsRule.JoinReduceExpressionsRule; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; @@ -118,6 +124,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveHepPlannerContext; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexExecutorImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveVolcanoPlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; @@ -865,10 +872,13 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu // Create MD provider HiveDefaultRelMetadataProvider mdProvider = new HiveDefaultRelMetadataProvider(conf); + + // Create executor + Executor executorProvider = new HiveRexExecutorImpl(cluster); // 2. Apply pre-join order optimizations calcitePreCboPlan = applyPreJoinOrderingTransforms(calciteGenPlan, - mdProvider.getMetadataProvider()); + mdProvider.getMetadataProvider(), executorProvider); // 3. Apply join order optimizations: reordering MST algorithm // If join optimizations failed because of missing stats, we continue with @@ -1011,9 +1021,10 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu * original plan * @param mdProvider * meta data provider + * @param executorProvider * @return */ - private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProvider mdProvider) { + private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProvider mdProvider, Executor executorProvider) { // TODO: Decorelation of subquery should be done before attempting // Partition Pruning; otherwise Expression evaluation may try to execute @@ -1053,11 +1064,20 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv basePlan = hepPlan(basePlan, true, mdProvider, HiveJoinAddNotNullRule.INSTANCE); } + // Note: we need to run Projection Pruning for ReduceExpressionsRule + // first. + HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, cluster, + HiveProject.DEFAULT_PROJECT_FACTORY, HiveFilter.DEFAULT_FILTER_FACTORY, + HiveJoin.HIVE_JOIN_FACTORY, HiveSemiJoin.HIVE_SEMIJOIN_FACTORY, + HiveSortLimit.HIVE_SORT_REL_FACTORY, HiveAggregate.HIVE_AGGR_REL_FACTORY, + HiveUnion.UNION_REL_FACTORY); + basePlan = fieldTrimmer.trim(basePlan); + // 4. Constant propagation, common filter extraction, and PPD - basePlan = hepPlan(basePlan, true, mdProvider, - ReduceExpressionsRule.PROJECT_INSTANCE, - ReduceExpressionsRule.FILTER_INSTANCE, - ReduceExpressionsRule.JOIN_INSTANCE, + basePlan = hepPlan(basePlan, true, mdProvider, executorProvider, + new ProjectReduceExpressionsRule(HiveProject.class, RelFactories.LOGICAL_BUILDER), + new FilterReduceExpressionsRule(HiveFilter.class, RelFactories.LOGICAL_BUILDER), + new JoinReduceExpressionsRule(HiveJoin.class, RelFactories.LOGICAL_BUILDER), HivePreFilteringRule.INSTANCE, new HiveFilterProjectTransposeRule(Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY, HiveProject.class, HiveProject.DEFAULT_PROJECT_FACTORY), @@ -1073,7 +1093,7 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv new HivePartitionPruneRule(conf)); // 6. Projection Pruning - HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, + fieldTrimmer = new HiveRelFieldTrimmer(null, cluster, HiveProject.DEFAULT_PROJECT_FACTORY, HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY, HiveSemiJoin.HIVE_SEMIJOIN_FACTORY, HiveSortLimit.HIVE_SORT_REL_FACTORY, @@ -1100,7 +1120,23 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv */ private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, RelMetadataProvider mdProvider, RelOptRule... rules) { - return hepPlan(basePlan, followPlanChanges, mdProvider, + return hepPlan(basePlan, followPlanChanges, mdProvider, null, + HepMatchOrder.TOP_DOWN, rules); + } + + /** + * Run the HEP Planner with the given rule set. + * + * @param basePlan + * @param followPlanChanges + * @param mdProvider + * @param executorProvider + * @param rules + * @return optimized RelNode + */ + private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, + RelMetadataProvider mdProvider, Executor executorProvider, RelOptRule... rules) { + return hepPlan(basePlan, followPlanChanges, mdProvider, executorProvider, HepMatchOrder.TOP_DOWN, rules); } @@ -1148,6 +1184,53 @@ private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, RelMetadata return optimizedRelNode; } + + /** + * Run the HEP Planner with the given rule set. + * + * @param basePlan + * @param followPlanChanges + * @param mdProvider + * @param executorProvider + * @param order + * @param rules + * @return optimized RelNode + */ + private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, + RelMetadataProvider mdProvider, Executor executorProvider, HepMatchOrder order, + RelOptRule... rules) { + + RelNode optimizedRelNode = basePlan; + HepProgramBuilder programBuilder = new HepProgramBuilder(); + if (followPlanChanges) { + programBuilder.addMatchOrder(order); + programBuilder = programBuilder.addRuleCollection(ImmutableList.copyOf(rules)); + } else { + // TODO: Should this be also TOP_DOWN? + for (RelOptRule r : rules) + programBuilder.addRuleInstance(r); + } + + HiveRulesRegistry registry = new HiveRulesRegistry(); + HiveHepPlannerContext context = new HiveHepPlannerContext(registry); + HepPlanner planner = new HepPlanner(programBuilder.build(), context); + List list = Lists.newArrayList(); + list.add(mdProvider); + planner.registerMetadataProviders(list); + RelMetadataProvider chainedProvider = ChainedRelMetadataProvider.of(list); + basePlan.getCluster().setMetadataProvider( + new CachingRelMetadataProvider(chainedProvider, planner)); + + // Executor is required for constant-reduction rules; see [CALCITE-566] + if (executorProvider != null) { + basePlan.getCluster().getPlanner().setExecutor(executorProvider); + } + + planner.setRoot(basePlan); + optimizedRelNode = planner.findBestExp(); + + return optimizedRelNode; + } @SuppressWarnings("nls") private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode leftRel, diff --git a/ql/src/test/queries/clientpositive/cbo_const.q b/ql/src/test/queries/clientpositive/cbo_const.q new file mode 100644 index 0000000..6409795 --- /dev/null +++ b/ql/src/test/queries/clientpositive/cbo_const.q @@ -0,0 +1,38 @@ +set hive.cbo.enable=false; + +EXPLAIN +SELECT x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key and y.key = 1+2) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11+3); + + +EXPLAIN +select key from (SELECT key from src where key = 1+3)s; + +select * from (select key from src where key = '1')subq; + +select '1'; + +select * from (select '1')subq; + +select * from (select key from src where false)subq; + +set hive.cbo.enable=true; + +EXPLAIN +SELECT x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key and y.key = 1+2) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11+3); + + +EXPLAIN +select key from (SELECT key from src where key = 1+3)s; + +select * from (select key from src where key = '1')subq; + +select '1'; + +select * from (select '1')subq; + +select * from (select key from src where false)subq; + diff --git a/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out b/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out index 20ccda5..50d5d8f 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out @@ -120,7 +120,7 @@ STAGE PLANS: alias: over1k Statistics: Num rows: 2098 Data size: 16736 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (((t = 1) and (si = 2)) or ((t = 2) and (si = 3)) or ((t = 3) and (si = 4)) or ((t = 4) and (si = 5)) or ((t = 5) and (si = 6)) or ((t = 6) and (si = 7)) or ((t = 7) and (si = 8)) or ((t = 9) and (si = 10)) or ((t = 10) and (si = 11)) or ((t = 11) and (si = 12)) or ((t = 12) and (si = 13)) or ((t = 13) and (si = 14)) or ((t = 14) and (si = 15)) or ((t = 15) and (si = 16)) or ((t = 16) and (si = 17)) or ((t = 17) and (si = 18)) or ((t = 27) and (si = 28)) or ((t = 37) and (si = 38)) or ((t = 47) and (si = 48)) or ((t = 52) and (si = 53))) (type: boolean) + predicate: ((((si = 2) or (si = 3) or (si = 4) or (si = 5) or (si = 6) or (si = 7) or (si = 8) or (si = 10) or (si = 11) or (si = 12) or (si = 13) or (si = 14) or (si = 15) or (si = 16) or (si = 17) or (si = 18) or (si = 28) or (si = 38) or (si = 48) or (si = 53)) and ((t = 1) or (t = 2) or (t = 3) or (t = 4) or (t = 5) or (t = 6) or (t = 7) or (t = 9) or (t = 10) or (t = 11) or (t = 12) or (t = 13) or (t = 14) or (t = 15) or (t = 16) or (t = 17) or (t = 27) or (t = 37) or (t = 47) or (t = 52))) and (((t = 1) and (si = 2)) or ((t = 2) and (si = 3)) or ((t = 3) and (si = 4)) or ((t = 4) and (si = 5)) or ((t = 5) and (si = 6)) or ((t = 6) and (si = 7)) or ((t = 7) and (si = 8)) or ((t = 9) and (si = 10)) or ((t = 10) and (si = 11)) or ((t = 11) and (si = 12)) or ((t = 12) and (si = 13)) or ((t = 13) and (si = 14)) or ((t = 14) and (si = 15)) or ((t = 15) and (si = 16)) or ((t = 16) and (si = 17)) or ((t = 17) and (si = 18)) or ((t = 27) and (si = 28)) or ((t = 37) and (si = 38)) or ((t = 47) and (si = 48)) or ((t = 52) and (si = 53)))) (type: boolean) Statistics: Num rows: 280 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 280 Data size: 2232 Basic stats: COMPLETE Column stats: COMPLETE @@ -209,7 +209,7 @@ STAGE PLANS: alias: over1k Statistics: Num rows: 2098 Data size: 211174 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((t = 1) and (si = 2)) or ((t = 2) and (si = 3)) or ((t = 3) and (si = 4)) or ((t = 4) and (si = 5)) or ((t = 5) and (si = 6)) or ((t = 6) and (si = 7)) or ((t = 7) and (si = 8)) or ((t = 9) and (si = 10)) or ((t = 10) and (si = 11)) or ((t = 11) and (si = 12)) or ((t = 12) and (si = 13)) or ((t = 13) and (si = 14)) or ((t = 14) and (si = 15)) or ((t = 15) and (si = 16)) or ((t = 16) and (si = 17)) or ((t = 17) and (si = 18)) or ((t = 27) and (si = 28)) or ((t = 37) and (si = 38)) or ((t = 47) and (si = 48)) or ((t = 52) and (si = 53))) (type: boolean) + predicate: ((((si = 2) or (si = 3) or (si = 4) or (si = 5) or (si = 6) or (si = 7) or (si = 8) or (si = 10) or (si = 11) or (si = 12) or (si = 13) or (si = 14) or (si = 15) or (si = 16) or (si = 17) or (si = 18) or (si = 28) or (si = 38) or (si = 48) or (si = 53)) and ((t = 1) or (t = 2) or (t = 3) or (t = 4) or (t = 5) or (t = 6) or (t = 7) or (t = 9) or (t = 10) or (t = 11) or (t = 12) or (t = 13) or (t = 14) or (t = 15) or (t = 16) or (t = 17) or (t = 27) or (t = 37) or (t = 47) or (t = 52))) and (((t = 1) and (si = 2)) or ((t = 2) and (si = 3)) or ((t = 3) and (si = 4)) or ((t = 4) and (si = 5)) or ((t = 5) and (si = 6)) or ((t = 6) and (si = 7)) or ((t = 7) and (si = 8)) or ((t = 9) and (si = 10)) or ((t = 10) and (si = 11)) or ((t = 11) and (si = 12)) or ((t = 12) and (si = 13)) or ((t = 13) and (si = 14)) or ((t = 14) and (si = 15)) or ((t = 15) and (si = 16)) or ((t = 16) and (si = 17)) or ((t = 17) and (si = 18)) or ((t = 27) and (si = 28)) or ((t = 37) and (si = 38)) or ((t = 47) and (si = 48)) or ((t = 52) and (si = 53)))) (type: boolean) Statistics: Num rows: 2098 Data size: 211174 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 2098 Data size: 211174 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out index 14e7e60..a34044b 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out @@ -426,12 +426,12 @@ STAGE PLANS: Select Operator expressions: ss_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 321 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 321 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE Reduce Operator Tree: Join Operator condition map: @@ -476,12 +476,12 @@ STAGE PLANS: Select Operator expressions: s_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: ss Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE @@ -556,12 +556,12 @@ STAGE PLANS: Select Operator expressions: ss_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 321 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 321 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE Reduce Operator Tree: Join Operator condition map: @@ -793,12 +793,12 @@ STAGE PLANS: Select Operator expressions: s_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: s Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE @@ -864,12 +864,12 @@ STAGE PLANS: Select Operator expressions: ss_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 321 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 321 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: s Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE @@ -909,10 +909,10 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) 2 _col0 (type: int) - outputColumnNames: _col2 + outputColumnNames: _col1 Statistics: Num rows: 273 Data size: 1092 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col2 (type: int) + expressions: _col1 (type: int) outputColumnNames: _col0 Statistics: Num rows: 273 Data size: 1092 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator diff --git a/ql/src/test/results/clientpositive/auto_join19.q.out b/ql/src/test/results/clientpositive/auto_join19.q.out index 8a57cb0..616aa77 100644 --- a/ql/src/test/results/clientpositive/auto_join19.q.out +++ b/ql/src/test/results/clientpositive/auto_join19.q.out @@ -65,10 +65,10 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col4 + outputColumnNames: _col0, _col2 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), _col4 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/auto_join9.q.out b/ql/src/test/results/clientpositive/auto_join9.q.out index 13dd5de..0c3f767 100644 --- a/ql/src/test/results/clientpositive/auto_join9.q.out +++ b/ql/src/test/results/clientpositive/auto_join9.q.out @@ -63,10 +63,10 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col4 + outputColumnNames: _col0, _col2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), _col4 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/cast1.q.out b/ql/src/test/results/clientpositive/cast1.q.out index 0bdecba..48a0c14 100644 --- a/ql/src/test/results/clientpositive/cast1.q.out +++ b/ql/src/test/results/clientpositive/cast1.q.out @@ -105,11 +105,11 @@ POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT 3 + 2, 3.0 + 2, 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 -POSTHOOK: Lineage: dest1.c1 EXPRESSION [] -POSTHOOK: Lineage: dest1.c2 EXPRESSION [] -POSTHOOK: Lineage: dest1.c3 EXPRESSION [] -POSTHOOK: Lineage: dest1.c4 EXPRESSION [] -POSTHOOK: Lineage: dest1.c5 EXPRESSION [] +POSTHOOK: Lineage: dest1.c1 SIMPLE [] +POSTHOOK: Lineage: dest1.c2 SIMPLE [] +POSTHOOK: Lineage: dest1.c3 SIMPLE [] +POSTHOOK: Lineage: dest1.c4 SIMPLE [] +POSTHOOK: Lineage: dest1.c5 SIMPLE [] POSTHOOK: Lineage: dest1.c6 EXPRESSION [] POSTHOOK: Lineage: dest1.c7 EXPRESSION [] PREHOOK: query: select dest1.* FROM dest1 diff --git a/ql/src/test/results/clientpositive/cbo_const.q.out b/ql/src/test/results/clientpositive/cbo_const.q.out new file mode 100644 index 0000000..2341666 --- /dev/null +++ b/ql/src/test/results/clientpositive/cbo_const.q.out @@ -0,0 +1,370 @@ +PREHOOK: query: EXPLAIN +SELECT x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key and y.key = 1+2) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11+3) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key and y.key = 1+2) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11+3) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 3) and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '3' (type: string) + sort order: + + Map-reduce partition columns: '3' (type: string) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = 3) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '3' (type: string) + sort order: + + Map-reduce partition columns: '3' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: _col1, _col6 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + value expressions: _col6 (type: string) + TableScan + alias: z + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (((ds = '2008-04-08') and (hr = 14)) and value is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 value (type: string) + outputColumnNames: _col6, _col11 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '3' (type: string), _col11 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +select key from (SELECT key from src where key = 1+3)s +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +select key from (SELECT key from src where key = 1+3)s +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = 4) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '4' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from (select key from src where key = '1')subq +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from (select key from src where key = '1')subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: select '1' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +1 +PREHOOK: query: select * from (select '1')subq +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select * from (select '1')subq +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +1 +PREHOOK: query: select * from (select key from src where false)subq +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from (select key from src where false)subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +SELECT x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key and y.key = 1+2) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11+3) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT x.key, z.value, y.value +FROM src1 x JOIN src y ON (x.key = y.key and y.key = 1+2) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11+3) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 3.0) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '3.0' (type: string) + sort order: + + Map-reduce partition columns: '3.0' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) = 3.0) and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '3.0' (type: string) + sort order: + + Map-reduce partition columns: '3.0' (type: string) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col3 (type: string) + sort order: + + Map-reduce partition columns: _col3 (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + TableScan + alias: z + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (((ds = '2008-04-08') and (UDFToDouble(hr) = 14.0)) and value is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col3 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col4 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '3.0' (type: string), _col4 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +select key from (SELECT key from src where key = 1+3)s +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +select key from (SELECT key from src where key = 1+3)s +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) = 4.0) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '4.0' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from (select key from src where key = '1')subq +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from (select key from src where key = '1')subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: select '1' +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +1 +PREHOOK: query: select * from (select '1')subq +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select * from (select '1')subq +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +1 +PREHOOK: query: select * from (select key from src where false)subq +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from (select key from src where false)subq +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out b/ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out index 09b981b..e938612 100644 --- a/ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out +++ b/ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out @@ -402,7 +402,7 @@ PREHOOK: query: select 3 * 5 from dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"753abad4d55afd3df34fdc73abfcd44d","queryText":"select 3 * 5 from dest1","edges":[{"sources":[],"targets":[0],"expression":"(3 * 5)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"_c0"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"753abad4d55afd3df34fdc73abfcd44d","queryText":"select 3 * 5 from dest1","edges":[{"sources":[],"targets":[0],"expression":"15","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"_c0"}]} 15 15 15 diff --git a/ql/src/test/results/clientpositive/create_genericudf.q.out b/ql/src/test/results/clientpositive/create_genericudf.q.out index 586f0ba..9484103 100644 --- a/ql/src/test/results/clientpositive/create_genericudf.q.out +++ b/ql/src/test/results/clientpositive/create_genericudf.q.out @@ -50,13 +50,13 @@ SELECT POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 -POSTHOOK: Lineage: dest1.c1 EXPRESSION [] -POSTHOOK: Lineage: dest1.c2 EXPRESSION [] -POSTHOOK: Lineage: dest1.c3 EXPRESSION [] -POSTHOOK: Lineage: dest1.c4 EXPRESSION [] -POSTHOOK: Lineage: dest1.c5 EXPRESSION [] -POSTHOOK: Lineage: dest1.c6 EXPRESSION [] -POSTHOOK: Lineage: dest1.c7 EXPRESSION [] +POSTHOOK: Lineage: dest1.c1 SIMPLE [] +POSTHOOK: Lineage: dest1.c2 SIMPLE [] +POSTHOOK: Lineage: dest1.c3 SIMPLE [] +POSTHOOK: Lineage: dest1.c4 SIMPLE [] +POSTHOOK: Lineage: dest1.c5 SIMPLE [] +POSTHOOK: Lineage: dest1.c6 SIMPLE [] +POSTHOOK: Lineage: dest1.c7 SIMPLE [] PREHOOK: query: SELECT dest1.* FROM dest1 LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 @@ -65,7 +65,7 @@ POSTHOOK: query: SELECT dest1.* FROM dest1 LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### -bbc bcc NULL NULL NULL bc abc +bbc bcc null null null bc abc PREHOOK: query: DROP TEMPORARY FUNCTION test_translate PREHOOK: type: DROPFUNCTION PREHOOK: Output: test_translate diff --git a/ql/src/test/results/clientpositive/cte_2.q.out b/ql/src/test/results/clientpositive/cte_2.q.out index a8bc760..d6923ba 100644 --- a/ql/src/test/results/clientpositive/cte_2.q.out +++ b/ql/src/test/results/clientpositive/cte_2.q.out @@ -40,7 +40,7 @@ select * POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@s1 -POSTHOOK: Lineage: s1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: s1.key SIMPLE [] POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: select * from s1 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/flatten_and_or.q.out b/ql/src/test/results/clientpositive/flatten_and_or.q.out index 9c51ff3..c1b4302 100644 --- a/ql/src/test/results/clientpositive/flatten_and_or.q.out +++ b/ql/src/test/results/clientpositive/flatten_and_or.q.out @@ -44,7 +44,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key = '0') and (value = '8')) or ((key = '1') and (value = '5')) or ((key = '2') and (value = '6')) or ((key = '3') and (value = '8')) or ((key = '4') and (value = '1')) or ((key = '5') and (value = '6')) or ((key = '6') and (value = '1')) or ((key = '7') and (value = '1')) or ((key = '8') and (value = '1')) or ((key = '9') and (value = '1')) or ((key = '10') and (value = '3'))) (type: boolean) + predicate: ((((value = '8') or (value = '5') or (value = '6') or (value = '1') or (value = '3')) and ((key = '0') or (key = '1') or (key = '2') or (key = '3') or (key = '4') or (key = '5') or (key = '6') or (key = '7') or (key = '8') or (key = '9') or (key = '10'))) and (((key = '0') and (value = '8')) or ((key = '1') and (value = '5')) or ((key = '2') and (value = '6')) or ((key = '3') and (value = '8')) or ((key = '4') and (value = '1')) or ((key = '5') and (value = '6')) or ((key = '6') and (value = '1')) or ((key = '7') and (value = '1')) or ((key = '8') and (value = '1')) or ((key = '9') and (value = '1')) or ((key = '10') and (value = '3')))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) diff --git a/ql/src/test/results/clientpositive/fouter_join_ppr.q.out b/ql/src/test/results/clientpositive/fouter_join_ppr.q.out index 56c5176..19f574e 100644 --- a/ql/src/test/results/clientpositive/fouter_join_ppr.q.out +++ b/ql/src/test/results/clientpositive/fouter_join_ppr.q.out @@ -1619,38 +1619,34 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 122 Data size: 1296 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false - predicate: ((UDFToDouble(_col3) > 15.0) and (UDFToDouble(_col3) < 25.0)) (type: boolean) + predicate: ((UDFToDouble(_col2) > 15.0) and (UDFToDouble(_col2) < 25.0)) (type: boolean) Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 1 + Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/groupby_ppd.q.out b/ql/src/test/results/clientpositive/groupby_ppd.q.out index 6164a26..e3e4a50 100644 --- a/ql/src/test/results/clientpositive/groupby_ppd.q.out +++ b/ql/src/test/results/clientpositive/groupby_ppd.q.out @@ -28,16 +28,16 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: foo (type: int) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Union Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: 1 (type: int), _col1 (type: int) - outputColumnNames: _col0, _col1 + expressions: _col0 (type: int) + outputColumnNames: _col1 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - keys: _col0 (type: int), _col1 (type: int) + keys: 1 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -54,16 +54,16 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: foo (type: int) - outputColumnNames: _col1 + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Union Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: 1 (type: int), _col1 (type: int) - outputColumnNames: _col0, _col1 + expressions: _col0 (type: int) + outputColumnNames: _col1 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - keys: _col0 (type: int), _col1 (type: int) + keys: 1 (type: int), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -79,7 +79,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col1 (type: int), _col0 (type: int) + expressions: _col1 (type: int), 1 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/input42.q.out b/ql/src/test/results/clientpositive/input42.q.out index 2974159..7658dc7 100644 --- a/ql/src/test/results/clientpositive/input42.q.out +++ b/ql/src/test/results/clientpositive/input42.q.out @@ -1724,14 +1724,53 @@ TOK_QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Partition Description: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (rand(100) < 0.1) (type: boolean) + Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### Partition + base file name: hr=11 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: @@ -1775,7 +1814,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.srcpart name: default.srcpart +#### A masked pattern was here #### Partition + base file name: hr=12 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: @@ -1819,20 +1860,15 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.srcpart name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [a] + /srcpart/ds=2008-04-08/hr=12 [a] + + Stage: Stage-0 + Fetch Operator + limit: -1 Processor Tree: - TableScan - alias: a - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (rand(100) < 0.1) (type: boolean) - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - ListSink + ListSink PREHOOK: query: select * from srcpart a where a.ds='2008-04-08' and rand(100) < 0.1 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/input_part6.q.out b/ql/src/test/results/clientpositive/input_part6.q.out index fa51cdf..c01d8af 100644 --- a/ql/src/test/results/clientpositive/input_part6.q.out +++ b/ql/src/test/results/clientpositive/input_part6.q.out @@ -19,7 +19,7 @@ STAGE PLANS: predicate: (UDFToDouble(ds) = 1996.0) (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string), '1996' (type: string), hr (type: string) + expressions: key (type: string), value (type: string), '1996.0' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Limit diff --git a/ql/src/test/results/clientpositive/insert1.q.out b/ql/src/test/results/clientpositive/insert1.q.out index 49dd2d5..7a2c429 100644 --- a/ql/src/test/results/clientpositive/insert1.q.out +++ b/ql/src/test/results/clientpositive/insert1.q.out @@ -26,7 +26,7 @@ POSTHOOK: query: insert overwrite table insert1 select a.key, a.value from inser POSTHOOK: type: QUERY POSTHOOK: Input: default@insert2 POSTHOOK: Output: default@insert1 -POSTHOOK: Lineage: insert1.key SIMPLE [(insert2)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: insert1.key SIMPLE [] POSTHOOK: Lineage: insert1.value SIMPLE [(insert2)a.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: explain insert into table insert1 select a.key, a.value from insert2 a WHERE (a.key=-1) PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/join28.q.out b/ql/src/test/results/clientpositive/join28.q.out index d748495..681d10f 100644 --- a/ql/src/test/results/clientpositive/join28.q.out +++ b/ql/src/test/results/clientpositive/join28.q.out @@ -109,10 +109,10 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col1, _col4 + outputColumnNames: _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: string), _col1 (type: string) + expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/join32.q.out b/ql/src/test/results/clientpositive/join32.q.out index afb373d..149a364 100644 --- a/ql/src/test/results/clientpositive/join32.q.out +++ b/ql/src/test/results/clientpositive/join32.q.out @@ -131,7 +131,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) Position of Big Table: 0 $hdt$_2:x @@ -174,20 +174,20 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3 + outputColumnNames: _col0, _col1 Position of Big Table: 0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 + outputColumnNames: _col0, _col1, _col4 Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/join32_lessSize.q.out b/ql/src/test/results/clientpositive/join32_lessSize.q.out index 8e71710..466504b 100644 --- a/ql/src/test/results/clientpositive/join32_lessSize.q.out +++ b/ql/src/test/results/clientpositive/join32_lessSize.q.out @@ -163,7 +163,7 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3 + outputColumnNames: _col0, _col1 Position of Big Table: 0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -175,7 +175,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col3 + columns _col0,_col1 columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -303,7 +303,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) Position of Big Table: 0 @@ -316,13 +316,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 + outputColumnNames: _col0, _col1, _col4 Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -362,7 +362,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col3 + columns _col0,_col1 columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -371,7 +371,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - columns _col0,_col3 + columns _col0,_col1 columns.types string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -1566,11 +1566,11 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 Position of Big Table: 1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2174,11 +2174,11 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 Position of Big Table: 1 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2556,10 +2556,10 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2813,10 +2813,10 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/join33.q.out b/ql/src/test/results/clientpositive/join33.q.out index afb373d..149a364 100644 --- a/ql/src/test/results/clientpositive/join33.q.out +++ b/ql/src/test/results/clientpositive/join33.q.out @@ -131,7 +131,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) Position of Big Table: 0 $hdt$_2:x @@ -174,20 +174,20 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3 + outputColumnNames: _col0, _col1 Position of Big Table: 0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 + outputColumnNames: _col0, _col1, _col4 Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/join9.q.out b/ql/src/test/results/clientpositive/join9.q.out index 8421036..e0ebae8 100644 --- a/ql/src/test/results/clientpositive/join9.q.out +++ b/ql/src/test/results/clientpositive/join9.q.out @@ -224,10 +224,10 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col4 + outputColumnNames: _col0, _col2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), _col4 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/lineage2.q.out b/ql/src/test/results/clientpositive/lineage2.q.out index 83785f2..b6d3d43 100644 --- a/ql/src/test/results/clientpositive/lineage2.q.out +++ b/ql/src/test/results/clientpositive/lineage2.q.out @@ -402,7 +402,7 @@ PREHOOK: query: select 3 * 5 from dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"753abad4d55afd3df34fdc73abfcd44d","queryText":"select 3 * 5 from dest1","edges":[{"sources":[],"targets":[0],"expression":"(3 * 5)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c0"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"753abad4d55afd3df34fdc73abfcd44d","queryText":"select 3 * 5 from dest1","edges":[{"sources":[],"targets":[0],"expression":"15","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c0"}]} 15 15 15 diff --git a/ql/src/test/results/clientpositive/lineage3.q.out b/ql/src/test/results/clientpositive/lineage3.q.out index fb5e9df..2b2fa89 100644 --- a/ql/src/test/results/clientpositive/lineage3.q.out +++ b/ql/src/test/results/clientpositive/lineage3.q.out @@ -166,7 +166,7 @@ where key in (select key+18 from src1) order by key PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"8b9d63653e36ecf4dd425d3cc3de9199","queryText":"select key, value from src1\nwhere key in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[],"targets":[0,1],"expression":"(1 = 1)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + UDFToDouble(18)))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"8b9d63653e36ecf4dd425d3cc3de9199","queryText":"select key, value from src1\nwhere key in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + UDFToDouble(18)))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"}]} 146 val_146 273 val_273 PREHOOK: query: select * from src1 a @@ -178,15 +178,15 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"8bf193b0658183be94e2428a79d91d10","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > UDFToDouble(300))","edgeType":"PREDICATE"},{"sources":[2,4],"targets":[0,1],"expression":"(UDFToDouble(a.key) = UDFToDouble((UDFToInteger(b.ctinyint) + 300)))","edgeType":"PREDICATE"},{"sources":[],"targets":[0,1],"expression":"(1 = 1)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"8bf193b0658183be94e2428a79d91d10","queryText":"select * from src1 a\nwhere exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > UDFToDouble(300))","edgeType":"PREDICATE"},{"sources":[2,4],"targets":[0,1],"expression":"(UDFToDouble(a.key) = UDFToDouble((UDFToInteger(b.ctinyint) + 300)))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 311 val_311 -Warning: Shuffle Join JOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select key, value from src1 where key not in (select key+18 from src1) order by key PREHOOK: type: QUERY PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"9b488fe1d7cf018aad3825173808cd36","queryText":"select key, value from src1\nwhere key not in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[],"targets":[0,1],"expression":"(1 = 1)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) + UDFToDouble(18)) is null","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"(count(*) = 0)","edgeType":"PREDICATE"},{"sources":[],"targets":[0,1],"expression":"true","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + UDFToDouble(18)))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"TABLE","vertexId":"default.src1"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"9b488fe1d7cf018aad3825173808cd36","queryText":"select key, value from src1\nwhere key not in (select key+18 from src1) order by key","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) + UDFToDouble(18)) is null","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"(count(*) = 0)","edgeType":"PREDICATE"},{"sources":[],"targets":[0,1],"expression":"true","edgeType":"PREDICATE"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(src1.key) = (UDFToDouble(src1.key) + UDFToDouble(18)))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"key"},{"id":1,"vertexType":"COLUMN","vertexId":"value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"TABLE","vertexId":"default.src1"}]} PREHOOK: query: select * from src1 a where not exists (select cint from alltypesorc b @@ -196,7 +196,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@src1 #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"53191056e05af9080a30de853e8cea9c","queryText":"select * from src1 a\nwhere not exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > UDFToDouble(300))","edgeType":"PREDICATE"},{"sources":[2,4],"targets":[0,1],"expression":"(UDFToDouble(a.key) = UDFToDouble((UDFToInteger(b.ctinyint) + 300)))","edgeType":"PREDICATE"},{"sources":[],"targets":[0,1],"expression":"(1 = 1)","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"(UDFToInteger(b.ctinyint) + 300) is null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"53191056e05af9080a30de853e8cea9c","queryText":"select * from src1 a\nwhere not exists\n (select cint from alltypesorc b\n where a.key = b.ctinyint + 300)\nand key > 300","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(UDFToDouble(a.key) > UDFToDouble(300))","edgeType":"PREDICATE"},{"sources":[2,4],"targets":[0,1],"expression":"(UDFToDouble(a.key) = UDFToDouble((UDFToInteger(b.ctinyint) + 300)))","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"(UDFToInteger(b.ctinyint) + 300) is null","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"a.key"},{"id":1,"vertexType":"COLUMN","vertexId":"a.value"},{"id":2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src1.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} 369 401 val_401 406 val_406 diff --git a/ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out b/ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out index 7f32108..c00e873 100644 --- a/ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out @@ -413,14 +413,99 @@ TOK_QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Partition Description: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: fact_daily + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((((value = 'val_484') or (value = 'val_238')) and ((key = '484') or (key = '238'))) and (((key = '484') and (value = 'val_484')) or ((key = '238') and (value = 'val_238')))) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1 + columns.types string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: value=val_238 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 1 + hr 4 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.fact_daily + numFiles 3 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct fact_daily { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.fact_daily + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct fact_daily { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.fact_daily + name: default.fact_daily +#### A masked pattern was here #### Partition + base file name: value=val_484 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: @@ -464,20 +549,15 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.fact_daily name: default.fact_daily + Truncated Path -> Alias: + /fact_daily/ds=1/hr=4/key=238/value=val_238 [$hdt$_0:fact_daily] + /fact_daily/ds=1/hr=4/key=484/value=val_484 [$hdt$_0:fact_daily] + + Stage: Stage-0 + Fetch Operator + limit: -1 Processor Tree: - TableScan - alias: fact_daily - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: (((key = '484') and (value = 'val_484')) or ((key = '238') and (value = 'val_238'))) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - ListSink + ListSink PREHOOK: query: -- List Bucketing Query SELECT key, value FROM fact_daily WHERE ds='1' and hr='4' and ( (key='484' and value ='val_484') or (key='238' and value= 'val_238')) diff --git a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out index d46b0ae..273234f 100644 --- a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out @@ -690,12 +690,10 @@ STAGE PLANS: predicate: (x = 484) (type: boolean) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 484 (type: int) - outputColumnNames: _col0 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - keys: _col0 (type: int) + keys: 484 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/literal_decimal.q.out b/ql/src/test/results/clientpositive/literal_decimal.q.out index eddc1a4..0b6299b 100644 --- a/ql/src/test/results/clientpositive/literal_decimal.q.out +++ b/ql/src/test/results/clientpositive/literal_decimal.q.out @@ -14,12 +14,12 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: -1 (type: decimal(1,0)), 0 (type: decimal(1,0)), 1 (type: decimal(1,0)), 3.14 (type: decimal(3,2)), -3.14 (type: decimal(3,2)), 99999999999999999 (type: decimal(17,0)), 99999999999999999.9999999999999 (type: decimal(30,13)), null (type: decimal(1,0)) + expressions: -1 (type: int), 0 (type: int), 1 (type: int), 3.14 (type: decimal(3,2)), -3.14 (type: decimal(3,2)), 99999999999999999 (type: bigint), 99999999999999999.9999999999999 (type: decimal(30,13)), null (type: void) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 500 Data size: 392000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 784 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: SELECT -1BD, 0BD, 1BD, 3.14BD, -3.14BD, 99999999999999999BD, 99999999999999999.9999999999999BD, 1E99BD FROM src LIMIT 1 diff --git a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out index 814c947..e72a494 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out @@ -1520,7 +1520,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 POSTHOOK: Output: default@tab@ds=2008-04-08 -POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [] POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: explain select count(*) @@ -1636,7 +1636,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 POSTHOOK: Output: default@tab@ds=2008-04-08 -POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [] POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: explain select count(*) diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index 68fa49c..589f47c 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -832,12 +832,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -855,7 +855,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -878,7 +878,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -973,12 +973,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Execution mode: llap Reducer 2 @@ -989,7 +989,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1928,7 +1928,6 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -1952,13 +1951,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Execution mode: llap Map 4 Map Operator Tree: @@ -1966,20 +1969,16 @@ STAGE PLANS: alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: ds + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: ds (type: string) - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap Reducer 2 Execution mode: llap @@ -1988,9 +1987,9 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 - 1 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + 0 _col0 (type: string) + 1 '2008-04-08' (type: string) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -2026,8 +2025,25 @@ STAGE PLANS: Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: + key expressions: '2008-04-08' (type: string) + sort order: + + Map-reduce partition columns: '2008-04-08' (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '2008-04-08' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target Input: srcpart + Partition key expr: ds + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Target column: ds + Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -2035,18 +2051,21 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -2106,12 +2125,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap Reducer 2 Execution mode: llap @@ -2122,10 +2141,10 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 = _col2) or (_col1 = _col4)) (type: boolean) + predicate: ((_col0 = _col2) or (_col1 = _col3)) (type: boolean) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -2224,12 +2243,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -2247,7 +2266,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -2270,7 +2289,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3985,7 +4004,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) input vertices: 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -4011,12 +4030,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -4034,7 +4053,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -4459,7 +4478,6 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -4482,13 +4500,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Execution mode: llap Map 2 Map Operator Tree: @@ -4496,20 +4518,16 @@ STAGE PLANS: alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: ds + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: ds (type: string) - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap Reducer 3 Execution mode: llap @@ -4525,11 +4543,12 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 - 1 + 0 _col0 (type: string) + 1 '2008-04-08' (type: string) input vertices: 0 Map 1 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -4561,18 +4580,21 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -5419,7 +5441,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), UDFToDouble(_col1) (type: double) - 1 _col0 (type: string), UDFToDouble(_col2) (type: double) + 1 _col0 (type: string), UDFToDouble(_col1) (type: double) input vertices: 1 Map 3 Statistics: Num rows: 1100 Data size: 103400 Basic stats: COMPLETE Column stats: NONE @@ -5445,12 +5467,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), UDFToDouble(_col2) (type: double) + key expressions: _col0 (type: string), UDFToDouble(_col1) (type: double) sort order: ++ - Map-reduce partition columns: _col0 (type: string), UDFToDouble(_col2) (type: double) + Map-reduce partition columns: _col0 (type: string), UDFToDouble(_col1) (type: double) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index 10b4168..3b546ef 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -832,12 +832,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -855,7 +855,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -878,7 +878,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -973,12 +973,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap Reducer 2 @@ -989,7 +989,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1928,7 +1928,6 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -1952,13 +1951,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Execution mode: llap Map 4 Map Operator Tree: @@ -1966,20 +1969,16 @@ STAGE PLANS: alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: ds + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: ds (type: string) - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap Reducer 2 Execution mode: llap @@ -1988,9 +1987,9 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 - 1 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + 0 _col0 (type: string) + 1 '2008-04-08' (type: string) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -2026,8 +2025,25 @@ STAGE PLANS: Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: + key expressions: '2008-04-08' (type: string) + sort order: + + Map-reduce partition columns: '2008-04-08' (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '2008-04-08' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target Input: srcpart + Partition key expr: ds + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Target column: ds + Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -2035,18 +2051,21 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -2106,12 +2125,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: vectorized, llap Reducer 2 Execution mode: llap @@ -2122,10 +2141,10 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 = _col2) or (_col1 = _col4)) (type: boolean) + predicate: ((_col0 = _col2) or (_col1 = _col3)) (type: boolean) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -2224,12 +2243,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -2247,7 +2266,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -2270,7 +2289,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3985,7 +4004,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) input vertices: 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -4011,12 +4030,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -4034,7 +4053,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -4459,7 +4478,6 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -4482,13 +4500,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Execution mode: llap Map 2 Map Operator Tree: @@ -4496,20 +4518,16 @@ STAGE PLANS: alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: ds + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: ds (type: string) - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap Reducer 3 Execution mode: vectorized, llap @@ -4525,11 +4543,12 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 - 1 + 0 _col0 (type: string) + 1 '2008-04-08' (type: string) input vertices: 0 Map 1 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -4561,18 +4580,21 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -5419,7 +5441,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), UDFToDouble(_col1) (type: double) - 1 _col0 (type: string), UDFToDouble(_col2) (type: double) + 1 _col0 (type: string), UDFToDouble(_col1) (type: double) input vertices: 1 Map 3 Statistics: Num rows: 1100 Data size: 103400 Basic stats: COMPLETE Column stats: NONE @@ -5445,12 +5467,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), UDFToDouble(_col2) (type: double) + key expressions: _col0 (type: string), UDFToDouble(_col1) (type: double) sort order: ++ - Map-reduce partition columns: _col0 (type: string), UDFToDouble(_col2) (type: double) + Map-reduce partition columns: _col0 (type: string), UDFToDouble(_col1) (type: double) Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) diff --git a/ql/src/test/results/clientpositive/mapjoin_subquery.q.out b/ql/src/test/results/clientpositive/mapjoin_subquery.q.out index 1f7a5f4..8a54caa 100644 --- a/ql/src/test/results/clientpositive/mapjoin_subquery.q.out +++ b/ql/src/test/results/clientpositive/mapjoin_subquery.q.out @@ -96,10 +96,10 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col1, _col4 + outputColumnNames: _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: string), _col1 (type: string) + expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -339,10 +339,10 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col1, _col4 + outputColumnNames: _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: string), _col1 (type: string) + expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/pcr.q.out b/ql/src/test/results/clientpositive/pcr.q.out index 4e23917..fbb7c8b 100644 --- a/ql/src/test/results/clientpositive/pcr.q.out +++ b/ql/src/test/results/clientpositive/pcr.q.out @@ -979,8 +979,8 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds=2000-04-08 [pcr_t1] - /pcr_t1/ds=2000-04-10 [pcr_t1] + /pcr_t1/ds=2000-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-10 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -1262,9 +1262,9 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds=2000-04-08 [pcr_t1] - /pcr_t1/ds=2000-04-09 [pcr_t1] - /pcr_t1/ds=2000-04-10 [pcr_t1] + /pcr_t1/ds=2000-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-09 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-10 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -1804,8 +1804,8 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds=2000-04-08 [pcr_t1] - /pcr_t1/ds=2000-04-09 [pcr_t1] + /pcr_t1/ds=2000-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-09 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -2010,8 +2010,8 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds=2000-04-08 [pcr_t1] - /pcr_t1/ds=2000-04-09 [pcr_t1] + /pcr_t1/ds=2000-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-09 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -2301,9 +2301,9 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds=2000-04-08 [pcr_t1] - /pcr_t1/ds=2000-04-09 [pcr_t1] - /pcr_t1/ds=2000-04-10 [pcr_t1] + /pcr_t1/ds=2000-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-09 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-10 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -2484,7 +2484,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (((ds = '2000-04-08') and (key = 1)) or ((ds = '2000-04-09') and (key = 2))) (type: boolean) + predicate: (((key = 1) or (key = 2)) and (((ds = '2000-04-08') and (key = 1)) or ((ds = '2000-04-09') and (key = 2)))) (type: boolean) Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds (type: string) @@ -2590,8 +2590,8 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds=2000-04-08 [pcr_t1] - /pcr_t1/ds=2000-04-09 [pcr_t1] + /pcr_t1/ds=2000-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-09 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -3574,10 +3574,10 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds=2000-04-08 [pcr_t1] - /pcr_t1/ds=2000-04-09 [pcr_t1] - /pcr_t1/ds=2000-04-10 [pcr_t1] - /pcr_t1/ds=2000-04-11 [pcr_t1] + /pcr_t1/ds=2000-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-09 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-10 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-11 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -3893,9 +3893,9 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds=2000-04-08 [pcr_t1] - /pcr_t1/ds=2000-04-09 [pcr_t1] - /pcr_t1/ds=2000-04-10 [pcr_t1] + /pcr_t1/ds=2000-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-09 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-10 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -5579,8 +5579,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Needs Tagging: false Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/pcs.q.out b/ql/src/test/results/clientpositive/pcs.q.out index 5cf0dff..80a2dbe 100644 --- a/ql/src/test/results/clientpositive/pcs.q.out +++ b/ql/src/test/results/clientpositive/pcs.q.out @@ -149,7 +149,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (struct(key,ds)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) (type: boolean) + predicate: (((key = 1) or (key = 2)) and (struct(key,ds)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09'))) (type: boolean) Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds (type: string) @@ -255,8 +255,8 @@ STAGE PLANS: name: default.pcs_t1 name: default.pcs_t1 Truncated Path -> Alias: - /pcs_t1/ds=2000-04-08 [pcs_t1] - /pcs_t1/ds=2000-04-09 [pcs_t1] + /pcs_t1/ds=2000-04-08 [$hdt$_0:pcs_t1] + /pcs_t1/ds=2000-04-09 [$hdt$_0:pcs_t1] Needs Tagging: false Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/pointlookup.q.out b/ql/src/test/results/clientpositive/pointlookup.q.out index a99b388..21bd22d 100644 --- a/ql/src/test/results/clientpositive/pointlookup.q.out +++ b/ql/src/test/results/clientpositive/pointlookup.q.out @@ -44,7 +44,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key = '0') and (value = '8')) or ((key = '1') and (value = '5')) or ((key = '2') and (value = '6')) or ((key = '3') and (value = '8')) or ((key = '4') and (value = '1')) or ((key = '5') and (value = '6')) or ((key = '6') and (value = '1')) or ((key = '7') and (value = '1')) or ((key = '8') and (value = '1')) or ((key = '9') and (value = '1')) or ((key = '10') and (value = '3'))) (type: boolean) + predicate: ((((value = '8') or (value = '5') or (value = '6') or (value = '1') or (value = '3')) and ((key = '0') or (key = '1') or (key = '2') or (key = '3') or (key = '4') or (key = '5') or (key = '6') or (key = '7') or (key = '8') or (key = '9') or (key = '10'))) and (((key = '0') and (value = '8')) or ((key = '1') and (value = '5')) or ((key = '2') and (value = '6')) or ((key = '3') and (value = '8')) or ((key = '4') and (value = '1')) or ((key = '5') and (value = '6')) or ((key = '6') and (value = '1')) or ((key = '7') and (value = '1')) or ((key = '8') and (value = '1')) or ((key = '9') and (value = '1')) or ((key = '10') and (value = '3')))) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) @@ -110,7 +110,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (struct(key,value)) IN (const struct('0','8'), const struct('1','5'), const struct('2','6'), const struct('3','8'), const struct('4','1'), const struct('5','6'), const struct('6','1'), const struct('7','1'), const struct('8','1'), const struct('9','1'), const struct('10','3')) (type: boolean) + predicate: ((((value = '8') or (value = '5') or (value = '6') or (value = '1') or (value = '3')) and ((key = '0') or (key = '1') or (key = '2') or (key = '3') or (key = '4') or (key = '5') or (key = '6') or (key = '7') or (key = '8') or (key = '9') or (key = '10'))) and (struct(key,value)) IN (const struct('0','8'), const struct('1','5'), const struct('2','6'), const struct('3','8'), const struct('4','1'), const struct('5','6'), const struct('6','1'), const struct('7','1'), const struct('8','1'), const struct('9','1'), const struct('10','3'))) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) @@ -176,7 +176,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (struct(key,value)) IN (const struct('0','8'), const struct('1','5'), const struct('2','6'), const struct('3','8'), const struct('4','1'), const struct('5','6'), const struct('6','1'), const struct('7','1'), const struct('8','1'), const struct('9','1'), const struct('10','3')) (type: boolean) + predicate: ((((value = '8') or (value = '5') or (value = '6') or (value = '1') or (value = '3')) and ((key = '0') or (key = '1') or (key = '2') or (key = '3') or (key = '4') or (key = '5') or (key = '6') or (key = '7') or (key = '8') or (key = '9') or (key = '10'))) and (struct(key,value)) IN (const struct('0','8'), const struct('1','5'), const struct('2','6'), const struct('3','8'), const struct('4','1'), const struct('5','6'), const struct('6','1'), const struct('7','1'), const struct('8','1'), const struct('9','1'), const struct('10','3'))) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) diff --git a/ql/src/test/results/clientpositive/pointlookup2.q.out b/ql/src/test/results/clientpositive/pointlookup2.q.out index b6270b3..44cdec0 100644 --- a/ql/src/test/results/clientpositive/pointlookup2.q.out +++ b/ql/src/test/results/clientpositive/pointlookup2.q.out @@ -68,7 +68,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@pcr_t1 POSTHOOK: Input: default@pcr_t1@ds=2000-04-08 POSTHOOK: Output: default@pcr_t2 -POSTHOOK: Lineage: pcr_t2.ds SIMPLE [(pcr_t1)pcr_t1.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: pcr_t2.ds SIMPLE [] POSTHOOK: Lineage: pcr_t2.key SIMPLE [(pcr_t1)pcr_t1.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: pcr_t2.value SIMPLE [(pcr_t1)pcr_t1.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: from pcr_t1 @@ -83,8 +83,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@pcr_t1 POSTHOOK: Input: default@pcr_t1@ds=2000-04-08 POSTHOOK: Output: default@pcr_t2 -POSTHOOK: Lineage: pcr_t2.ds SIMPLE [(pcr_t1)pcr_t1.FieldSchema(name:ds, type:string, comment:null), ] -POSTHOOK: Lineage: pcr_t2.key SIMPLE [(pcr_t1)pcr_t1.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: pcr_t2.ds SIMPLE [] +POSTHOOK: Lineage: pcr_t2.key SIMPLE [] POSTHOOK: Lineage: pcr_t2.value SIMPLE [(pcr_t1)pcr_t1.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: explain extended select key, value, ds @@ -165,7 +165,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (struct(key,ds)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09')) (type: boolean) + predicate: (((key = 1) or (key = 2)) and (struct(key,ds)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09'))) (type: boolean) Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds (type: string) @@ -271,8 +271,8 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds=2000-04-08 [pcr_t1] - /pcr_t1/ds=2000-04-09 [pcr_t1] + /pcr_t1/ds=2000-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds=2000-04-09 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -878,7 +878,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain extended select * from pcr_t1 t1 join pcr_t2 t2 @@ -1140,9 +1140,9 @@ STAGE PLANS: name: default.pcr_t2 name: default.pcr_t2 Truncated Path -> Alias: - /pcr_t1/ds=2000-04-08 [$hdt$_0:t1] - /pcr_t1/ds=2000-04-09 [$hdt$_0:t1] - /pcr_t2 [$hdt$_1:t2] + /pcr_t1/ds=2000-04-08 [$hdt$_0:$hdt$_0:t1] + /pcr_t1/ds=2000-04-09 [$hdt$_0:$hdt$_0:t1] + /pcr_t2 [$hdt$_1:$hdt$_1:t2] Needs Tagging: true Reduce Operator Tree: Join Operator @@ -1247,7 +1247,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain extended select * from pcr_t1 t1 join pcr_t2 t2 @@ -1554,10 +1554,10 @@ STAGE PLANS: name: default.pcr_t2 name: default.pcr_t2 Truncated Path -> Alias: - /pcr_t1/ds=2000-04-08 [$hdt$_0:t1] - /pcr_t1/ds=2000-04-09 [$hdt$_0:t1] - /pcr_t1/ds=2000-04-10 [$hdt$_0:t1] - /pcr_t2 [$hdt$_1:t2] + /pcr_t1/ds=2000-04-08 [$hdt$_0:$hdt$_0:t1] + /pcr_t1/ds=2000-04-09 [$hdt$_0:$hdt$_0:t1] + /pcr_t1/ds=2000-04-10 [$hdt$_0:$hdt$_0:t1] + /pcr_t2 [$hdt$_1:$hdt$_1:t2] Needs Tagging: true Reduce Operator Tree: Join Operator diff --git a/ql/src/test/results/clientpositive/pointlookup3.q.out b/ql/src/test/results/clientpositive/pointlookup3.q.out index a37e079..33c6c13 100644 --- a/ql/src/test/results/clientpositive/pointlookup3.q.out +++ b/ql/src/test/results/clientpositive/pointlookup3.q.out @@ -125,7 +125,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (struct(ds1,key)) IN (const struct('2000-04-08',1), const struct('2000-04-09',2)) (type: boolean) + predicate: (((key = 1) or (key = 2)) and (struct(key,ds1)) IN (const struct(1,'2000-04-08'), const struct(2,'2000-04-09'))) (type: boolean) Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds1 (type: string), ds2 (type: string) @@ -233,8 +233,8 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds1=2000-04-08/ds2=2001-04-08 [pcr_t1] - /pcr_t1/ds1=2000-04-09/ds2=2001-04-09 [pcr_t1] + /pcr_t1/ds1=2000-04-08/ds2=2001-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds1=2000-04-09/ds2=2001-04-09 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -365,7 +365,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key = 1) (type: boolean) + predicate: (((key = 1) or (key = 2)) and (key = 1)) (type: boolean) Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string), ds1 (type: string) @@ -427,7 +427,7 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds1=2000-04-08/ds2=2001-04-08 [pcr_t1] + /pcr_t1/ds1=2000-04-08/ds2=2001-04-08 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -1036,7 +1036,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain extended select * from pcr_t1 t1 join pcr_t1 t2 @@ -1302,9 +1302,9 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds1=2000-04-08/ds2=2001-04-08 [$hdt$_0:t1, $hdt$_1:t1] - /pcr_t1/ds1=2000-04-09/ds2=2001-04-09 [$hdt$_0:t1, $hdt$_1:t1] - /pcr_t1/ds1=2000-04-10/ds2=2001-04-10 [$hdt$_1:t1] + /pcr_t1/ds1=2000-04-08/ds2=2001-04-08 [$hdt$_0:$hdt$_0:t1, $hdt$_1:$hdt$_1:t1] + /pcr_t1/ds1=2000-04-09/ds2=2001-04-09 [$hdt$_0:$hdt$_0:t1, $hdt$_1:$hdt$_1:t1] + /pcr_t1/ds1=2000-04-10/ds2=2001-04-10 [$hdt$_1:$hdt$_1:t1] Needs Tagging: true Reduce Operator Tree: Join Operator diff --git a/ql/src/test/results/clientpositive/pointlookup4.q.out b/ql/src/test/results/clientpositive/pointlookup4.q.out index 157aea6..cdd14ff 100644 --- a/ql/src/test/results/clientpositive/pointlookup4.q.out +++ b/ql/src/test/results/clientpositive/pointlookup4.q.out @@ -135,7 +135,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (((ds1 = '2000-04-08') and (ds2 = '2001-04-08') and (key = 1)) or ((ds1 = '2000-04-09') and (ds2 = '2001-04-09') and (key = 2))) (type: boolean) + predicate: (((key = 1) or (key = 2)) and (((ds1 = '2000-04-08') and (ds2 = '2001-04-08') and (key = 1)) or ((ds1 = '2000-04-09') and (ds2 = '2001-04-09') and (key = 2)))) (type: boolean) Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds1 (type: string), ds2 (type: string) @@ -243,8 +243,8 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds1=2000-04-08/ds2=2001-04-08 [pcr_t1] - /pcr_t1/ds1=2000-04-09/ds2=2001-04-09 [pcr_t1] + /pcr_t1/ds1=2000-04-08/ds2=2001-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds1=2000-04-09/ds2=2001-04-09 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator @@ -375,7 +375,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (struct(ds1,key,ds2)) IN (const struct('2000-04-08',1,'2001-04-08'), const struct('2000-04-09',2,'2001-04-09')) (type: boolean) + predicate: (((key = 1) or (key = 2)) and (struct(key,ds2,ds1)) IN (const struct(1,'2001-04-08','2000-04-08'), const struct(2,'2001-04-09','2000-04-09'))) (type: boolean) Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds1 (type: string), ds2 (type: string) @@ -483,8 +483,8 @@ STAGE PLANS: name: default.pcr_t1 name: default.pcr_t1 Truncated Path -> Alias: - /pcr_t1/ds1=2000-04-08/ds2=2001-04-08 [pcr_t1] - /pcr_t1/ds1=2000-04-09/ds2=2001-04-09 [pcr_t1] + /pcr_t1/ds1=2000-04-08/ds2=2001-04-08 [$hdt$_0:pcr_t1] + /pcr_t1/ds1=2000-04-09/ds2=2001-04-09 [$hdt$_0:pcr_t1] Needs Tagging: false Reduce Operator Tree: Select Operator diff --git a/ql/src/test/results/clientpositive/quotedid_basic.q.out b/ql/src/test/results/clientpositive/quotedid_basic.q.out index 50c83a8..96d8819 100644 --- a/ql/src/test/results/clientpositive/quotedid_basic.q.out +++ b/ql/src/test/results/clientpositive/quotedid_basic.q.out @@ -101,11 +101,11 @@ STAGE PLANS: predicate: (!@#$%^&*()_q = '1') (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: x+1 (type: string), y&y (type: string), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: x+1 (type: string), y&y (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string), _col2 (type: string) + keys: _col0 (type: string), _col1 (type: string), '1' (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -156,11 +156,11 @@ STAGE PLANS: predicate: (!@#$%^&*()_q = '1') (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: x+1 (type: string), y&y (type: string), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: x+1 (type: string), y&y (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string), _col2 (type: string) + keys: _col0 (type: string), _col1 (type: string), '1' (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -187,27 +187,27 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: string), _col1 (type: string) + key expressions: '1' (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col2 (type: string) + Map-reduce partition columns: '1' (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col0 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE PTF Operator Function definitions: Input definition input alias: ptf_0 - output shape: _col0: string, _col1: string, _col2: string + output shape: _col0: string, _col1: string type: WINDOWING Windowing table definition input alias: ptf_1 name: windowingtablefunction order by: _col1 - partition by: _col2 + partition by: '1' raw input shape: window functions: window function definition @@ -219,7 +219,7 @@ STAGE PLANS: isPivotResult: true Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), rank_window_0 (type: int) + expressions: _col0 (type: string), _col1 (type: string), '1' (type: string), rank_window_0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator @@ -260,11 +260,11 @@ STAGE PLANS: predicate: (!@#$%^&*()_q = '1') (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: x+1 (type: string), y&y (type: string), '1' (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: x+1 (type: string), y&y (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string), _col2 (type: string) + keys: _col0 (type: string), _col1 (type: string), '1' (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -291,27 +291,27 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col2 (type: string), _col1 (type: string) + key expressions: '1' (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col2 (type: string) + Map-reduce partition columns: '1' (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col0 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE PTF Operator Function definitions: Input definition input alias: ptf_0 - output shape: _col0: string, _col1: string, _col2: string + output shape: _col0: string, _col1: string type: WINDOWING Windowing table definition input alias: ptf_1 name: windowingtablefunction order by: _col1 - partition by: _col2 + partition by: '1' raw input shape: window functions: window function definition @@ -323,7 +323,7 @@ STAGE PLANS: isPivotResult: true Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), rank_window_0 (type: int) + expressions: _col0 (type: string), _col1 (type: string), '1' (type: string), rank_window_0 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/quotedid_partition.q.out b/ql/src/test/results/clientpositive/quotedid_partition.q.out index bc52c82..30cfce3 100644 --- a/ql/src/test/results/clientpositive/quotedid_partition.q.out +++ b/ql/src/test/results/clientpositive/quotedid_partition.q.out @@ -46,11 +46,11 @@ STAGE PLANS: predicate: (x+1 = '10') (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '10' (type: string), y&y (type: string), 'a' (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: y&y (type: string) + outputColumnNames: _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string), _col2 (type: string) + keys: '10' (type: string), _col1 (type: string), 'a' (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out b/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out index 634e171..4db38a7 100644 --- a/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out +++ b/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out @@ -57,14 +57,53 @@ TOK_QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Partition Description: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((rand(1) < 0.1) and ((UDFToDouble(key) <= 50.0) and (UDFToDouble(key) >= 10.0))) (type: boolean) + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### Partition + base file name: hr=12 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: @@ -108,20 +147,14 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.srcpart name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=12 [a] + + Stage: Stage-0 + Fetch Operator + limit: -1 Processor Tree: - TableScan - alias: a - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: ((rand(1) < 0.1) and ((UDFToDouble(key) <= 50.0) and (UDFToDouble(key) >= 10.0))) (type: boolean) - Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE - ListSink + ListSink PREHOOK: query: select a.* from srcpart a where rand(1) < 0.1 and a.ds = '2008-04-08' and not(key > 50 or key < 10) and a.hr like '%2' PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/router_join_ppr.q.out b/ql/src/test/results/clientpositive/router_join_ppr.q.out index 4d3f81d..dccbe98 100644 --- a/ql/src/test/results/clientpositive/router_join_ppr.q.out +++ b/ql/src/test/results/clientpositive/router_join_ppr.q.out @@ -780,38 +780,34 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 122 Data size: 1296 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false predicate: ((UDFToDouble(_col0) > 10.0) and (UDFToDouble(_col0) < 20.0)) (type: boolean) Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -1532,34 +1528,30 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 1 + Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_18.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_18.q.out index 6106188..4b29056 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin_18.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin_18.q.out @@ -238,17 +238,28 @@ STAGE PLANS: predicate: (key = 238) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), value (type: string) + expressions: 238 (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table2 + value expressions: _col1 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table2 Stage: Stage-0 Move Operator @@ -277,7 +288,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table1 POSTHOOK: Input: default@test_table1@ds=1 POSTHOOK: Output: default@test_table2@ds=2 -POSTHOOK: Lineage: test_table2 PARTITION(ds=2).key SIMPLE [(test_table1)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: test_table2 PARTITION(ds=2).key SIMPLE [] POSTHOOK: Lineage: test_table2 PARTITION(ds=2).value SIMPLE [(test_table1)a.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: select count(*) from test_table2 where ds = '2' PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/spark/auto_join19.q.out b/ql/src/test/results/clientpositive/spark/auto_join19.q.out index 9e4fb8f..42072b9 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join19.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join19.q.out @@ -68,12 +68,12 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col4 + outputColumnNames: _col0, _col2 input vertices: 1 Map 2 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), _col4 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/auto_join9.q.out b/ql/src/test/results/clientpositive/spark/auto_join9.q.out index 568891b..91251e8 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join9.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join9.q.out @@ -66,12 +66,12 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col4 + outputColumnNames: _col0, _col2 input vertices: 1 Map 2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), _col4 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out index fa80956..1ab695b 100644 --- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out @@ -1560,7 +1560,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 POSTHOOK: Output: default@tab@ds=2008-04-08 -POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [] POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: explain select count(*) @@ -1681,7 +1681,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 POSTHOOK: Output: default@tab@ds=2008-04-08 -POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [] POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: explain select count(*) diff --git a/ql/src/test/results/clientpositive/spark/insert1.q.out b/ql/src/test/results/clientpositive/spark/insert1.q.out index e72ba16..50e8376 100644 --- a/ql/src/test/results/clientpositive/spark/insert1.q.out +++ b/ql/src/test/results/clientpositive/spark/insert1.q.out @@ -26,7 +26,7 @@ POSTHOOK: query: insert overwrite table insert1 select a.key, a.value from inser POSTHOOK: type: QUERY POSTHOOK: Input: default@insert2 POSTHOOK: Output: default@insert1 -POSTHOOK: Lineage: insert1.key SIMPLE [(insert2)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: insert1.key SIMPLE [] POSTHOOK: Lineage: insert1.value SIMPLE [(insert2)a.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: explain insert into table insert1 select a.key, a.value from insert2 a WHERE (a.key=-1) PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/spark/join28.q.out b/ql/src/test/results/clientpositive/spark/join28.q.out index a8177f3..c2dc48e 100644 --- a/ql/src/test/results/clientpositive/spark/join28.q.out +++ b/ql/src/test/results/clientpositive/spark/join28.q.out @@ -114,12 +114,12 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col1, _col4 + outputColumnNames: _col1, _col2 input vertices: 0 Map 1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: string), _col1 (type: string) + expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/join32.q.out b/ql/src/test/results/clientpositive/spark/join32.q.out index 1c1c103..8e157f2 100644 --- a/ql/src/test/results/clientpositive/spark/join32.q.out +++ b/ql/src/test/results/clientpositive/spark/join32.q.out @@ -196,7 +196,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) Position of Big Table: 0 Local Work: @@ -275,7 +275,7 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3 + outputColumnNames: _col0, _col1 input vertices: 1 Map 2 Position of Big Table: 0 @@ -284,15 +284,15 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 + outputColumnNames: _col0, _col1, _col4 input vertices: 1 Map 3 Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out b/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out index e2ae796..3b19f3c 100644 --- a/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out +++ b/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out @@ -204,7 +204,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) Position of Big Table: 0 Local Work: @@ -283,7 +283,7 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3 + outputColumnNames: _col0, _col1 input vertices: 1 Map 2 Position of Big Table: 0 @@ -292,15 +292,15 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 + outputColumnNames: _col0, _col1, _col4 input vertices: 1 Map 3 Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1420,13 +1420,13 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 input vertices: 0 Map 1 Position of Big Table: 1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -1942,13 +1942,13 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 input vertices: 0 Map 1 Position of Big Table: 1 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2289,12 +2289,12 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 input vertices: 0 Map 1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -2533,12 +2533,12 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3, _col4 + outputColumnNames: _col0, _col1, _col2 input vertices: 0 Map 1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/join33.q.out b/ql/src/test/results/clientpositive/spark/join33.q.out index 1c1c103..8e157f2 100644 --- a/ql/src/test/results/clientpositive/spark/join33.q.out +++ b/ql/src/test/results/clientpositive/spark/join33.q.out @@ -196,7 +196,7 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) Position of Big Table: 0 Local Work: @@ -275,7 +275,7 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col3 + outputColumnNames: _col0, _col1 input vertices: 1 Map 2 Position of Big Table: 0 @@ -284,15 +284,15 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 _col3 (type: string) + 0 _col1 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col3, _col6 + outputColumnNames: _col0, _col1, _col4 input vertices: 1 Map 3 Position of Big Table: 0 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col3 (type: string), _col0 (type: string), _col6 (type: string) + expressions: _col1 (type: string), _col0 (type: string), _col4 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/join9.q.out b/ql/src/test/results/clientpositive/spark/join9.q.out index c7440da..899b404 100644 --- a/ql/src/test/results/clientpositive/spark/join9.q.out +++ b/ql/src/test/results/clientpositive/spark/join9.q.out @@ -236,10 +236,10 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col4 + outputColumnNames: _col0, _col2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToInteger(_col0) (type: int), _col4 (type: string) + expressions: UDFToInteger(_col0) (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_subquery.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_subquery.q.out index d74b7d0..d1b2301 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin_subquery.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin_subquery.q.out @@ -101,12 +101,12 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col1, _col4 + outputColumnNames: _col1, _col2 input vertices: 0 Map 1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: string), _col1 (type: string) + expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -351,12 +351,12 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col1, _col4 + outputColumnNames: _col1, _col2 input vertices: 0 Map 1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: string), _col1 (type: string) + expressions: _col2 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git a/ql/src/test/results/clientpositive/spark/pcr.q.out b/ql/src/test/results/clientpositive/spark/pcr.q.out index 82593d4..ae40738 100644 --- a/ql/src/test/results/clientpositive/spark/pcr.q.out +++ b/ql/src/test/results/clientpositive/spark/pcr.q.out @@ -2543,7 +2543,7 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (((ds = '2000-04-08') and (key = 1)) or ((ds = '2000-04-09') and (key = 2))) (type: boolean) + predicate: (((key = 1) or (key = 2)) and (((ds = '2000-04-08') and (key = 1)) or ((ds = '2000-04-09') and (key = 2)))) (type: boolean) Statistics: Num rows: 40 Data size: 320 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds (type: string) diff --git a/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out b/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out index 6e34865..173132f 100644 --- a/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out +++ b/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out @@ -804,38 +804,34 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 122 Data size: 1296 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false predicate: ((UDFToDouble(_col0) > 10.0) and (UDFToDouble(_col0) < 20.0)) (type: boolean) Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Fetch Operator @@ -1580,34 +1576,30 @@ STAGE PLANS: keys: 0 _col0 (type: string) 1 _col0 (type: string) - outputColumnNames: _col0, _col1, _col3, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 1 + Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out index 3076e06..bfdd529 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out @@ -233,6 +233,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -244,17 +246,29 @@ STAGE PLANS: predicate: (key = 238) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), value (type: string) + expressions: 238 (type: int), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table2 + value expressions: _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table2 Stage: Stage-0 Move Operator @@ -283,7 +297,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table1 POSTHOOK: Input: default@test_table1@ds=1 POSTHOOK: Output: default@test_table2@ds=2 -POSTHOOK: Lineage: test_table2 PARTITION(ds=2).key SIMPLE [(test_table1)a.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: test_table2 PARTITION(ds=2).key SIMPLE [] POSTHOOK: Lineage: test_table2 PARTITION(ds=2).value SIMPLE [(test_table1)a.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: select count(*) from test_table2 where ds = '2' PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out index babaff8..17821a8 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out @@ -150,7 +150,7 @@ POSTHOOK: query: insert overwrite table smb_bucket4_1 select * from smb_bucket_i POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_input POSTHOOK: Output: default@smb_bucket4_1 -POSTHOOK: Lineage: smb_bucket4_1.key SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_bucket4_1.key SIMPLE [] POSTHOOK: Lineage: smb_bucket4_1.value SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=5000 PREHOOK: type: QUERY @@ -160,7 +160,7 @@ POSTHOOK: query: insert overwrite table smb_bucket4_2 select * from smb_bucket_i POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_input POSTHOOK: Output: default@smb_bucket4_2 -POSTHOOK: Lineage: smb_bucket4_2.key SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_bucket4_2.key SIMPLE [] POSTHOOK: Lineage: smb_bucket4_2.value SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key PREHOOK: type: QUERY @@ -335,7 +335,7 @@ POSTHOOK: query: insert overwrite table smb_bucket4_1 select * from smb_bucket_i POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_input POSTHOOK: Output: default@smb_bucket4_1 -POSTHOOK: Lineage: smb_bucket4_1.key SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_bucket4_1.key SIMPLE [] POSTHOOK: Lineage: smb_bucket4_1.value SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=5000 PREHOOK: type: QUERY @@ -345,7 +345,7 @@ POSTHOOK: query: insert overwrite table smb_bucket4_2 select * from smb_bucket_i POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_input POSTHOOK: Output: default@smb_bucket4_2 -POSTHOOK: Lineage: smb_bucket4_2.key SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_bucket4_2.key SIMPLE [] POSTHOOK: Lineage: smb_bucket4_2.value SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: insert overwrite table smb_bucket4_3 select * from smb_bucket_input where key=4000 PREHOOK: type: QUERY @@ -355,7 +355,7 @@ POSTHOOK: query: insert overwrite table smb_bucket4_3 select * from smb_bucket_i POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_input POSTHOOK: Output: default@smb_bucket4_3 -POSTHOOK: Lineage: smb_bucket4_3.key SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_bucket4_3.key SIMPLE [] POSTHOOK: Lineage: smb_bucket4_3.value SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: select /*+mapjoin(b,c)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key full outer join smb_bucket4_3 c on a.key=c.key @@ -381,7 +381,7 @@ POSTHOOK: query: insert overwrite table smb_bucket4_1 select * from smb_bucket_i POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_input POSTHOOK: Output: default@smb_bucket4_1 -POSTHOOK: Lineage: smb_bucket4_1.key SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_bucket4_1.key SIMPLE [] POSTHOOK: Lineage: smb_bucket4_1.value SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=4000 PREHOOK: type: QUERY @@ -391,7 +391,7 @@ POSTHOOK: query: insert overwrite table smb_bucket4_2 select * from smb_bucket_i POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_input POSTHOOK: Output: default@smb_bucket4_2 -POSTHOOK: Lineage: smb_bucket4_2.key SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_bucket4_2.key SIMPLE [] POSTHOOK: Lineage: smb_bucket4_2.value SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: insert overwrite table smb_bucket4_3 select * from smb_bucket_input where key=5000 PREHOOK: type: QUERY @@ -401,7 +401,7 @@ POSTHOOK: query: insert overwrite table smb_bucket4_3 select * from smb_bucket_i POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_input POSTHOOK: Output: default@smb_bucket4_3 -POSTHOOK: Lineage: smb_bucket4_3.key SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_bucket4_3.key SIMPLE [] POSTHOOK: Lineage: smb_bucket4_3.value SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: select /*+mapjoin(b,c)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key full outer join smb_bucket4_3 c on a.key=c.key @@ -427,7 +427,7 @@ POSTHOOK: query: insert overwrite table smb_bucket4_1 select * from smb_bucket_i POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_input POSTHOOK: Output: default@smb_bucket4_1 -POSTHOOK: Lineage: smb_bucket4_1.key SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_bucket4_1.key SIMPLE [] POSTHOOK: Lineage: smb_bucket4_1.value SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=4000 PREHOOK: type: QUERY @@ -437,7 +437,7 @@ POSTHOOK: query: insert overwrite table smb_bucket4_2 select * from smb_bucket_i POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_input POSTHOOK: Output: default@smb_bucket4_2 -POSTHOOK: Lineage: smb_bucket4_2.key SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_bucket4_2.key SIMPLE [] POSTHOOK: Lineage: smb_bucket4_2.value SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: insert overwrite table smb_bucket4_3 select * from smb_bucket_input where key=5000 PREHOOK: type: QUERY @@ -447,7 +447,7 @@ POSTHOOK: query: insert overwrite table smb_bucket4_3 select * from smb_bucket_i POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_input POSTHOOK: Output: default@smb_bucket4_3 -POSTHOOK: Lineage: smb_bucket4_3.key SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: smb_bucket4_3.key SIMPLE [] POSTHOOK: Lineage: smb_bucket4_3.value SIMPLE [(smb_bucket_input)smb_bucket_input.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: select /*+mapjoin(b,c)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key full outer join smb_bucket4_3 c on a.key=c.key diff --git a/ql/src/test/results/clientpositive/spark/union_date_trim.q.out b/ql/src/test/results/clientpositive/spark/union_date_trim.q.out index e2f5269..324e8b7 100644 --- a/ql/src/test/results/clientpositive/spark/union_date_trim.q.out +++ b/ql/src/test/results/clientpositive/spark/union_date_trim.q.out @@ -51,4 +51,4 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@testdate POSTHOOK: Output: default@testdate POSTHOOK: Lineage: testdate.dt EXPRESSION [(testdate)testdate.FieldSchema(name:dt, type:date, comment:null), ] -POSTHOOK: Lineage: testdate.id EXPRESSION [(testdate)testdate.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: testdate.id EXPRESSION [] diff --git a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out index cef6138..427db9f 100644 --- a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out @@ -122,12 +122,12 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2, _col4 + outputColumnNames: _col2, _col3 input vertices: 1 Reducer 4 Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: int), _col2 (type: int) + expressions: _col3 (type: int), _col2 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -201,21 +201,21 @@ STAGE PLANS: alias: lineitem Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((l_shipmode = 'AIR') and (l_linenumber = 1)) and l_orderkey is not null) (type: boolean) + predicate: (((l_shipmode = 'AIR') and l_orderkey is not null) and l_linenumber is not null) (type: boolean) Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: l_orderkey (type: int) - outputColumnNames: _col0 + expressions: l_orderkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: int) + keys: _col0 (type: int), _col1 (type: int) mode: hash - outputColumnNames: _col0 + outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + 0 _col0 (type: int), 1 (type: int) + 1 _col0 (type: int), _col1 (type: int) Local Work: Map Reduce Local Work @@ -239,8 +239,8 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + 0 _col0 (type: int), 1 (type: int) + 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col1, _col2 input vertices: 1 Map 2 diff --git a/ql/src/test/results/clientpositive/stats_empty_partition.q.out b/ql/src/test/results/clientpositive/stats_empty_partition.q.out index c13817e..202263e 100644 --- a/ql/src/test/results/clientpositive/stats_empty_partition.q.out +++ b/ql/src/test/results/clientpositive/stats_empty_partition.q.out @@ -20,7 +20,7 @@ POSTHOOK: query: insert overwrite table tmptable partition (part = '1') select * POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@tmptable@part=1 -POSTHOOK: Lineage: tmptable PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmptable PARTITION(part=1).key SIMPLE [] POSTHOOK: Lineage: tmptable PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: describe formatted tmptable partition (part = '1') PREHOOK: type: DESCTABLE diff --git a/ql/src/test/results/clientpositive/subquery_notin.q.out b/ql/src/test/results/clientpositive/subquery_notin.q.out index 2973e1f..bb91306 100644 --- a/ql/src/test/results/clientpositive/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/subquery_notin.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- non agg, non corr explain select * @@ -151,7 +151,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from src where src.key not in ( select key from src s1 where s1.key > '2') @@ -285,7 +285,7 @@ POSTHOOK: Input: default@src 199 val_199 199 val_199 2 val_2 -Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- non agg, corr explain select p_mfgr, b.p_name, p_size @@ -530,7 +530,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select p_mfgr, b.p_name, p_size from part b where b.p_name not in @@ -569,7 +569,7 @@ Manufacturer#4 almond azure aquamarine papaya violet 12 Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#5 almond aquamarine dodger light gainsboro 46 Manufacturer#5 almond azure blanched chiffon midnight 23 -Warning: Shuffle Join JOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- agg, non corr explain select p_name, p_size @@ -851,7 +851,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[39][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select p_name, p_size from part where part.p_size not in @@ -898,7 +898,7 @@ almond aquamarine sandy cyan gainsboro 18 almond aquamarine yellow dodger mint 7 almond azure aquamarine papaya violet 12 almond azure blanched chiffon midnight 23 -Warning: Shuffle Join JOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- agg, corr explain select p_mfgr, p_name, p_size @@ -1212,7 +1212,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[38][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select p_mfgr, p_name, p_size from part b where b.p_size not in (select min(p_size) @@ -1288,7 +1288,7 @@ POSTHOOK: Input: default@lineitem 139636 1 175839 1 182052 1 -Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- alternate not in syntax select * from src @@ -1452,7 +1452,7 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@t1_v POSTHOOK: Output: database:default POSTHOOK: Output: default@T2_v -Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) @@ -1597,7 +1597,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from T1_v where T1_v.key not in (select T2_v.key from T2_v) PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out b/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out index c08e2b9..1e087d2 100644 --- a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out +++ b/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join JOIN[24][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[23][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: -- non agg, non corr -- JAVA_VERSION_SPECIFIC_OUTPUT @@ -188,7 +188,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: -- non agg, corr explain select b.p_mfgr, min(p_retailprice) @@ -445,7 +445,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: select b.p_mfgr, min(p_retailprice) from part b group by b.p_mfgr @@ -470,7 +470,7 @@ POSTHOOK: Input: default@part #### A masked pattern was here #### Manufacturer#1 1173.15 Manufacturer#2 1690.68 -Warning: Shuffle Join JOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: -- agg, non corr explain select b.p_mfgr, min(p_retailprice) @@ -737,7 +737,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: select b.p_mfgr, min(p_retailprice) from part b group by b.p_mfgr diff --git a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out index c09d327..87aaed2 100644 --- a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out +++ b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out @@ -775,7 +775,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[29][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: -- non agg, corr explain select p_mfgr, b.p_name, p_size diff --git a/ql/src/test/results/clientpositive/subquery_views.q.out b/ql/src/test/results/clientpositive/subquery_views.q.out index 470fa83..2afd7ff 100644 --- a/ql/src/test/results/clientpositive/subquery_views.q.out +++ b/ql/src/test/results/clientpositive/subquery_views.q.out @@ -111,8 +111,8 @@ where `b`.`key` not in from `default`.`src` `a` where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_11' ), tableType:VIRTUAL_VIEW) -Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -Warning: Shuffle Join JOIN[50][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[46][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product PREHOOK: query: explain select * from cv2 where cv2.key in (select key from cv2 c where c.key < '11') @@ -420,8 +420,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -Warning: Shuffle Join JOIN[50][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product +Warning: Shuffle Join JOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[46][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product PREHOOK: query: select * from cv2 where cv2.key in (select key from cv2 c where c.key < '11') PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out index 2447f19..90bc6c0 100644 --- a/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out +++ b/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out @@ -1482,7 +1482,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 POSTHOOK: Output: default@tab@ds=2008-04-08 -POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [] POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: explain select count(*) @@ -1595,7 +1595,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 POSTHOOK: Output: default@tab@ds=2008-04-08 -POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [] POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: explain select count(*) diff --git a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out index d95fe2a..7b6b87c 100644 --- a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out @@ -809,12 +809,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -832,7 +832,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -853,7 +853,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -946,12 +946,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: @@ -960,7 +960,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1870,7 +1870,6 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -1894,42 +1893,42 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: ds + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: ds (type: string) - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Merge Join Operator condition map: Inner Join 0 to 1 keys: - 0 - 1 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + 0 _col0 (type: string) + 1 '2008-04-08' (type: string) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1963,8 +1962,25 @@ STAGE PLANS: Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: + key expressions: '2008-04-08' (type: string) + sort order: + + Map-reduce partition columns: '2008-04-08' (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '2008-04-08' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target Input: srcpart + Partition key expr: ds + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Target column: ds + Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -1972,18 +1988,21 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -2042,12 +2061,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -2056,10 +2075,10 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 = _col2) or (_col1 = _col4)) (type: boolean) + predicate: ((_col0 = _col2) or (_col1 = _col3)) (type: boolean) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -2156,12 +2175,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -2179,7 +2198,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -2200,7 +2219,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3860,7 +3879,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) input vertices: 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -3885,12 +3904,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -3908,7 +3927,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -4322,7 +4341,6 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -4345,33 +4363,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 2 Map Operator Tree: TableScan alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: ds + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: ds (type: string) - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reducer 3 Reduce Operator Tree: Group By Operator @@ -4385,11 +4403,12 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 - 1 + 0 _col0 (type: string) + 1 '2008-04-08' (type: string) input vertices: 0 Map 1 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -4420,18 +4439,21 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -5255,7 +5277,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), UDFToDouble(_col1) (type: double) - 1 _col0 (type: string), UDFToDouble(_col2) (type: double) + 1 _col0 (type: string), UDFToDouble(_col1) (type: double) input vertices: 1 Map 3 Statistics: Num rows: 1100 Data size: 103400 Basic stats: COMPLETE Column stats: NONE @@ -5280,12 +5302,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), UDFToDouble(_col2) (type: double) + key expressions: _col0 (type: string), UDFToDouble(_col1) (type: double) sort order: ++ - Map-reduce partition columns: _col0 (type: string), UDFToDouble(_col2) (type: double) + Map-reduce partition columns: _col0 (type: string), UDFToDouble(_col1) (type: double) Statistics: Num rows: 1 Data size: 27 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) diff --git a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out index a3d1f87..7fcb040 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out @@ -2588,11 +2588,11 @@ Stage-0 | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ - | Statistics:Num rows: 5 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + | Statistics:Num rows: 5 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col1 (type: int) | Select Operator [SEL_2] | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 5 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE + | Statistics:Num rows: 5 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE | Filter Operator [FIL_16] | predicate:((((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0))) and key is not null) (type: boolean) | Statistics:Num rows: 5 Data size: 372 Basic stats: COMPLETE Column stats: COMPLETE @@ -2915,7 +2915,7 @@ Stage-0 limit:-1 Select Operator [SEL_2] outputColumnNames:["_col0"] - Filter Operator [FIL_4] + Filter Operator [FIL_6] predicate:((c_int = -6) or (c_int = 6)) (type: boolean) TableScan [TS_0] alias:cbo_t1 @@ -2935,48 +2935,45 @@ Stage-0 limit:-1 Stage-1 Reducer 3 - File Output Operator [FS_15] + File Output Operator [FS_16] compressed:false Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Group By Operator [GBY_13] + Group By Operator [GBY_14] | aggregations:["count(VALUE._col0)"] | outputColumnNames:["_col0"] | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 2 [SIMPLE_EDGE] - Reduce Output Operator [RS_12] + Reduce Output Operator [RS_13] sort order: Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions:_col0 (type: bigint) - Group By Operator [GBY_11] - aggregations:["count(_col0)"] + Group By Operator [GBY_12] + aggregations:["count('2014')"] outputColumnNames:["_col0"] Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_10] - outputColumnNames:["_col0"] - Statistics:Num rows: 200 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Merge Join Operator [MERGEJOIN_19] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{} - | Statistics:Num rows: 200 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_7] - | sort order: - | Statistics:Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_2] - | Statistics:Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_0] - | alias:cbo_t1 - | Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE - |<-Map 4 [SIMPLE_EDGE] - Reduce Output Operator [RS_8] - sort order: + Merge Join Operator [MERGEJOIN_20] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{} + | Statistics:Num rows: 200 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + |<-Map 1 [SIMPLE_EDGE] + | Reduce Output Operator [RS_7] + | sort order: + | Statistics:Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + | Select Operator [SEL_2] + | Statistics:Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + | TableScan [TS_0] + | alias:cbo_t1 + | Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE + |<-Map 4 [SIMPLE_EDGE] + Reduce Output Operator [RS_8] + sort order: + Statistics:Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator [SEL_5] Statistics:Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_5] - Statistics:Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_3] - alias:cbo_t2 - Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE + TableScan [TS_3] + alias:cbo_t2 + Statistics:Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: COMPLETE PREHOOK: query: explain select * from src_cbo b @@ -3005,61 +3002,61 @@ Stage-0 limit:-1 Stage-1 Reducer 2 - File Output Operator [FS_15] + File Output Operator [FS_14] compressed:false Statistics:Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Select Operator [SEL_14] + Select Operator [SEL_13] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_17] + Filter Operator [FIL_16] predicate:_col3 is null (type: boolean) Statistics:Num rows: 1 Data size: 269 Basic stats: COMPLETE Column stats: COMPLETE - Merge Join Operator [MERGEJOIN_19] + Merge Join Operator [MERGEJOIN_18] | condition map:[{"":"Left Outer Join0 to 1"}] | keys:{"0":"_col1 (type: string)","1":"_col1 (type: string)"} | outputColumnNames:["_col0","_col1","_col3"] | Statistics:Num rows: 193 Data size: 51917 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_10] + | Reduce Output Operator [RS_9] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) | sort order:+ | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col0 (type: string) - | Select Operator [SEL_2] + | Select Operator [SEL_1] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE | TableScan [TS_0] | alias:b | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 4 [SIMPLE_EDGE] - Reduce Output Operator [RS_11] + Reduce Output Operator [RS_10] key expressions:_col1 (type: string) Map-reduce partition columns:_col1 (type: string) sort order:+ Statistics:Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_9] + Select Operator [SEL_8] outputColumnNames:["_col1"] Statistics:Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_8] + Group By Operator [GBY_7] | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_7] + Reduce Output Operator [RS_6] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_6] + Group By Operator [GBY_5] keys:key (type: string), value (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_18] + Filter Operator [FIL_17] predicate:(value > 'val_2') (type: boolean) Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_3] + TableScan [TS_2] alias:b Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE @@ -3092,57 +3089,57 @@ Stage-0 limit:-1 Stage-1 Reducer 3 - File Output Operator [FS_15] + File Output Operator [FS_14] compressed:false Statistics:Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Select Operator [SEL_14] + Select Operator [SEL_13] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_17] + Filter Operator [FIL_16] predicate:_col3 is null (type: boolean) Statistics:Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE - Merge Join Operator [MERGEJOIN_19] + Merge Join Operator [MERGEJOIN_18] | condition map:[{"":"Left Outer Join0 to 1"}] | keys:{"0":"_col1 (type: string), _col0 (type: string)","1":"_col0 (type: string), _col1 (type: string)"} | outputColumnNames:["_col0","_col1","_col3"] | Statistics:Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 4 [SIMPLE_EDGE] - | Reduce Output Operator [RS_11] + | Reduce Output Operator [RS_10] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_9] + | Select Operator [SEL_8] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_18] + | Filter Operator [FIL_17] | predicate:(value > 'val_12') (type: boolean) | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_7] + | TableScan [TS_6] | alias:b | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 2 [SIMPLE_EDGE] - Reduce Output Operator [RS_10] + Reduce Output Operator [RS_9] key expressions:_col1 (type: string), _col0 (type: string) Map-reduce partition columns:_col1 (type: string), _col0 (type: string) sort order:++ Statistics:Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_5] + Group By Operator [GBY_4] | keys:KEY._col0 (type: string), KEY._col1 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 1 [SIMPLE_EDGE] - Reduce Output Operator [RS_4] + Reduce Output Operator [RS_3] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_3] + Group By Operator [GBY_2] keys:key (type: string), value (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_2] + Select Operator [SEL_1] outputColumnNames:["key","value"] Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE TableScan [TS_0] @@ -3185,17 +3182,17 @@ Stage-0 limit:-1 Stage-1 Reducer 2 - File Output Operator [FS_16] + File Output Operator [FS_14] compressed:false Statistics:Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Merge Join Operator [MERGEJOIN_21] + Merge Join Operator [MERGEJOIN_19] | condition map:[{"":"Left Semi Join 0 to 1"}] | keys:{"0":"_col1 (type: string), _col0 (type: string)","1":"_col0 (type: string), _col1 (type: string)"} | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_11] + | Reduce Output Operator [RS_9] | key expressions:_col1 (type: string), _col0 (type: string) | Map-reduce partition columns:_col1 (type: string), _col0 (type: string) | sort order:++ @@ -3203,29 +3200,29 @@ Stage-0 | Select Operator [SEL_2] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_19] + | Filter Operator [FIL_17] | predicate:((value > 'val_9') and key is not null) (type: boolean) | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE | TableScan [TS_0] | alias:b | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_13] + Reduce Output Operator [RS_11] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_9] + Group By Operator [GBY_7] keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_7] + Select Operator [SEL_5] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_20] + Filter Operator [FIL_18] predicate:((value > 'val_9') and key is not null) (type: boolean) Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_5] + TableScan [TS_3] alias:b Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE @@ -3257,17 +3254,17 @@ Stage-0 limit:-1 Stage-1 Reducer 2 - File Output Operator [FS_16] + File Output Operator [FS_14] compressed:false Statistics:Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Merge Join Operator [MERGEJOIN_21] + Merge Join Operator [MERGEJOIN_19] | condition map:[{"":"Left Semi Join 0 to 1"}] | keys:{"0":"_col1 (type: string), _col0 (type: string)","1":"_col0 (type: string), _col1 (type: string)"} | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_11] + | Reduce Output Operator [RS_9] | key expressions:_col1 (type: string), _col0 (type: string) | Map-reduce partition columns:_col1 (type: string), _col0 (type: string) | sort order:++ @@ -3275,29 +3272,29 @@ Stage-0 | Select Operator [SEL_2] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_19] + | Filter Operator [FIL_17] | predicate:((value > 'val_9') and key is not null) (type: boolean) | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE | TableScan [TS_0] | alias:b | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_13] + Reduce Output Operator [RS_11] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_9] + Group By Operator [GBY_7] keys:_col0 (type: string), _col1 (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_7] + Select Operator [SEL_5] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_20] + Filter Operator [FIL_18] predicate:((value > 'val_9') and key is not null) (type: boolean) Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_5] + TableScan [TS_3] alias:b Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE @@ -3319,17 +3316,17 @@ Stage-0 limit:-1 Stage-1 Reducer 2 - File Output Operator [FS_16] + File Output Operator [FS_14] compressed:false Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Merge Join Operator [MERGEJOIN_21] + Merge Join Operator [MERGEJOIN_19] | condition map:[{"":"Left Semi Join 0 to 1"}] | keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"} | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_11] + | Reduce Output Operator [RS_9] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ @@ -3338,29 +3335,29 @@ Stage-0 | Select Operator [SEL_2] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_19] + | Filter Operator [FIL_17] | predicate:(key > '9') (type: boolean) | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE | TableScan [TS_0] | alias:src_cbo | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_13] + Reduce Output Operator [RS_11] key expressions:_col0 (type: string) Map-reduce partition columns:_col0 (type: string) sort order:+ Statistics:Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_9] + Group By Operator [GBY_7] keys:_col0 (type: string) outputColumnNames:["_col0"] Statistics:Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_7] + Select Operator [SEL_5] outputColumnNames:["_col0"] Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_20] + Filter Operator [FIL_18] predicate:(key > '9') (type: boolean) Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_5] + TableScan [TS_3] alias:src_cbo Statistics:Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE @@ -3386,89 +3383,89 @@ Stage-0 limit:-1 Stage-1 Reducer 3 - File Output Operator [FS_27] + File Output Operator [FS_25] compressed:false Statistics:Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Select Operator [SEL_26] + Select Operator [SEL_24] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Merge Join Operator [MERGEJOIN_37] + Merge Join Operator [MERGEJOIN_35] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{"0":"_col1 (type: int)","1":"_col0 (type: int)"} | outputColumnNames:["_col2","_col4"] | Statistics:Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 2 [SIMPLE_EDGE] - | Reduce Output Operator [RS_22] + | Reduce Output Operator [RS_20] | key expressions:_col1 (type: int) | Map-reduce partition columns:_col1 (type: int) | sort order:+ | Statistics:Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col2 (type: int) - | Merge Join Operator [MERGEJOIN_36] + | Merge Join Operator [MERGEJOIN_34] | | condition map:[{"":"Left Semi Join 0 to 1"}] - | | keys:{"0":"_col0 (type: int)","1":"_col0 (type: int)"} + | | keys:{"0":"_col0 (type: int), 1 (type: int)","1":"_col0 (type: int), _col1 (type: int)"} | | outputColumnNames:["_col1","_col2"] | | Statistics:Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE | |<-Map 1 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_17] - | | key expressions:_col0 (type: int) - | | Map-reduce partition columns:_col0 (type: int) - | | sort order:+ + | | Reduce Output Operator [RS_15] + | | key expressions:_col0 (type: int), 1 (type: int) + | | Map-reduce partition columns:_col0 (type: int), 1 (type: int) + | | sort order:++ | | Statistics:Num rows: 16 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE | | value expressions:_col1 (type: int), _col2 (type: int) | | Select Operator [SEL_2] | | outputColumnNames:["_col0","_col1","_col2"] | | Statistics:Num rows: 16 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE - | | Filter Operator [FIL_33] + | | Filter Operator [FIL_31] | | predicate:(((l_linenumber = 1) and l_orderkey is not null) and l_partkey is not null) (type: boolean) | | Statistics:Num rows: 16 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE | | TableScan [TS_0] | | alias:lineitem | | Statistics:Num rows: 100 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE | |<-Map 4 [SIMPLE_EDGE] - | Reduce Output Operator [RS_19] - | key expressions:_col0 (type: int) - | Map-reduce partition columns:_col0 (type: int) - | sort order:+ - | Statistics:Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - | Group By Operator [GBY_15] - | keys:_col0 (type: int) - | outputColumnNames:["_col0"] - | Statistics:Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + | Reduce Output Operator [RS_17] + | key expressions:_col0 (type: int), _col1 (type: int) + | Map-reduce partition columns:_col0 (type: int), _col1 (type: int) + | sort order:++ + | Statistics:Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + | Group By Operator [GBY_13] + | keys:_col0 (type: int), _col1 (type: int) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE | Select Operator [SEL_5] - | outputColumnNames:["_col0"] - | Statistics:Num rows: 14 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_34] - | predicate:(((l_shipmode = 'AIR') and (l_linenumber = 1)) and l_orderkey is not null) (type: boolean) + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 14 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + | Filter Operator [FIL_32] + | predicate:(((l_shipmode = 'AIR') and l_orderkey is not null) and l_linenumber is not null) (type: boolean) | Statistics:Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE | TableScan [TS_3] | alias:lineitem | Statistics:Num rows: 100 Data size: 9600 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 6 [SIMPLE_EDGE] - Reduce Output Operator [RS_24] + Reduce Output Operator [RS_22] key expressions:_col0 (type: int) Map-reduce partition columns:_col0 (type: int) sort order:+ Statistics:Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_12] + Group By Operator [GBY_10] | keys:KEY._col0 (type: int) | outputColumnNames:["_col0"] | Statistics:Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 5 [SIMPLE_EDGE] - Reduce Output Operator [RS_11] + Reduce Output Operator [RS_9] key expressions:_col0 (type: int) Map-reduce partition columns:_col0 (type: int) sort order:+ Statistics:Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_10] + Group By Operator [GBY_8] keys:l_partkey (type: int) outputColumnNames:["_col0"] Statistics:Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_35] + Filter Operator [FIL_33] predicate:l_partkey is not null (type: boolean) Statistics:Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_8] + TableScan [TS_6] alias:lineitem Statistics:Num rows: 100 Data size: 400 Basic stats: COMPLETE Column stats: COMPLETE @@ -3497,49 +3494,49 @@ Stage-0 limit:-1 Stage-1 Reducer 4 - File Output Operator [FS_37] + File Output Operator [FS_33] compressed:false Statistics:Num rows: 34 Data size: 6324 Basic stats: COMPLETE Column stats: COMPLETE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Merge Join Operator [MERGEJOIN_50] + Merge Join Operator [MERGEJOIN_46] | condition map:[{"":"Left Semi Join 0 to 1"}] | keys:{"0":"_col2 (type: bigint)","1":"_col0 (type: bigint)"} | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 34 Data size: 6324 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 3 [SIMPLE_EDGE] - | Reduce Output Operator [RS_32] + | Reduce Output Operator [RS_28] | key expressions:_col2 (type: bigint) | Map-reduce partition columns:_col2 (type: bigint) | sort order:+ | Statistics:Num rows: 83 Data size: 15438 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col0 (type: string), _col1 (type: string) - | Filter Operator [FIL_43] + | Filter Operator [FIL_39] | predicate:_col2 is not null (type: boolean) | Statistics:Num rows: 83 Data size: 15438 Basic stats: COMPLETE Column stats: COMPLETE - | Group By Operator [GBY_20] + | Group By Operator [GBY_16] | | aggregations:["count(VALUE._col0)"] | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1","_col2"] | | Statistics:Num rows: 83 Data size: 15438 Basic stats: COMPLETE Column stats: COMPLETE | |<-Reducer 2 [SIMPLE_EDGE] - | Reduce Output Operator [RS_19] + | Reduce Output Operator [RS_15] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 83 Data size: 15438 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col2 (type: bigint) - | Group By Operator [GBY_18] + | Group By Operator [GBY_14] | aggregations:["count()"] | keys:_col0 (type: string), _col1 (type: string) | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 83 Data size: 15438 Basic stats: COMPLETE Column stats: COMPLETE - | Merge Join Operator [MERGEJOIN_49] + | Merge Join Operator [MERGEJOIN_45] | | condition map:[{"":"Left Semi Join 0 to 1"}] | | keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE | |<-Map 1 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_13] + | | Reduce Output Operator [RS_9] | | key expressions:_col0 (type: string) | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ @@ -3548,71 +3545,71 @@ Stage-0 | | Select Operator [SEL_2] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - | | Filter Operator [FIL_44] + | | Filter Operator [FIL_40] | | predicate:(key > '8') (type: boolean) | | Statistics:Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE | | TableScan [TS_0] | | alias:b | | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE | |<-Map 5 [SIMPLE_EDGE] - | Reduce Output Operator [RS_15] + | Reduce Output Operator [RS_11] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE - | Group By Operator [GBY_11] + | Group By Operator [GBY_7] | keys:_col0 (type: string) | outputColumnNames:["_col0"] | Statistics:Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_9] + | Select Operator [SEL_5] | outputColumnNames:["_col0"] | Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_45] + | Filter Operator [FIL_41] | predicate:(key > '8') (type: boolean) | Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_7] + | TableScan [TS_3] | alias:b | Statistics:Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 7 [SIMPLE_EDGE] - Reduce Output Operator [RS_34] + Reduce Output Operator [RS_30] key expressions:_col0 (type: bigint) Map-reduce partition columns:_col0 (type: bigint) sort order:+ Statistics:Num rows: 34 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_30] + Group By Operator [GBY_26] keys:_col0 (type: bigint) outputColumnNames:["_col0"] Statistics:Num rows: 34 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_28] + Select Operator [SEL_24] outputColumnNames:["_col0"] Statistics:Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_46] + Filter Operator [FIL_42] predicate:_col1 is not null (type: boolean) Statistics:Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_48] + Select Operator [SEL_44] outputColumnNames:["_col1"] Statistics:Num rows: 69 Data size: 552 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_27] + Group By Operator [GBY_23] | aggregations:["count(VALUE._col0)"] | keys:KEY._col0 (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 6 [SIMPLE_EDGE] - Reduce Output Operator [RS_26] + Reduce Output Operator [RS_22] key expressions:_col0 (type: string) Map-reduce partition columns:_col0 (type: string) sort order:+ Statistics:Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE value expressions:_col1 (type: bigint) - Group By Operator [GBY_25] + Group By Operator [GBY_21] aggregations:["count()"] keys:key (type: string) outputColumnNames:["_col0","_col1"] Statistics:Num rows: 69 Data size: 6555 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_47] + Filter Operator [FIL_43] predicate:(key > '9') (type: boolean) Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_22] + TableScan [TS_18] alias:b Statistics:Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE @@ -3640,78 +3637,78 @@ Stage-0 limit:-1 Stage-1 Reducer 3 - File Output Operator [FS_21] + File Output Operator [FS_20] compressed:false Statistics:Num rows: 3 Data size: 681 Basic stats: COMPLETE Column stats: COMPLETE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Merge Join Operator [MERGEJOIN_26] + Merge Join Operator [MERGEJOIN_25] | condition map:[{"":"Left Semi Join 0 to 1"}] | keys:{"0":"_col1 (type: string)","1":"_col0 (type: string)"} | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 3 Data size: 681 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 2 [SIMPLE_EDGE] - | Reduce Output Operator [RS_16] + | Reduce Output Operator [RS_15] | key expressions:_col1 (type: string) | Map-reduce partition columns:_col1 (type: string) | sort order:+ | Statistics:Num rows: 13 Data size: 2951 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col0 (type: string), _col2 (type: double) - | Select Operator [SEL_6] + | Select Operator [SEL_5] | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 13 Data size: 2951 Basic stats: COMPLETE Column stats: COMPLETE - | Group By Operator [GBY_5] + | Group By Operator [GBY_4] | | aggregations:["avg(VALUE._col0)"] | | keys:KEY._col0 (type: string), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1","_col2"] | | Statistics:Num rows: 13 Data size: 2951 Basic stats: COMPLETE Column stats: COMPLETE | |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_4] + | Reduce Output Operator [RS_3] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 13 Data size: 2847 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col2 (type: struct) - | Group By Operator [GBY_3] + | Group By Operator [GBY_2] | aggregations:["avg(p_size)"] | keys:p_name (type: string), p_mfgr (type: string) | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 13 Data size: 2847 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_24] + | Filter Operator [FIL_23] | predicate:p_name is not null (type: boolean) | Statistics:Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE | TableScan [TS_0] | alias:part | Statistics:Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 5 [SIMPLE_EDGE] - Reduce Output Operator [RS_18] + Reduce Output Operator [RS_17] key expressions:_col0 (type: string) Map-reduce partition columns:_col0 (type: string) sort order:+ Statistics:Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_14] + Group By Operator [GBY_13] keys:_col0 (type: string) outputColumnNames:["_col0"] Statistics:Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_11] + Select Operator [SEL_10] outputColumnNames:["_col0"] Statistics:Num rows: 13 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_25] + Filter Operator [FIL_24] predicate:first_value_window_0 is not null (type: boolean) Statistics:Num rows: 13 Data size: 6383 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator [PTF_10] + PTF Operator [PTF_9] Function definitions:[{"Input definition":{"type:":"WINDOWING"}},{"name:":"windowingtablefunction","order by:":"_col5","partition by:":"_col2"}] Statistics:Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_9] + Select Operator [SEL_8] | outputColumnNames:["_col1","_col2","_col5"] | Statistics:Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 4 [SIMPLE_EDGE] - Reduce Output Operator [RS_8] + Reduce Output Operator [RS_7] key expressions:p_mfgr (type: string), p_size (type: int) Map-reduce partition columns:p_mfgr (type: string) sort order:++ Statistics:Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE value expressions:p_name (type: string) - TableScan [TS_7] + TableScan [TS_6] alias:part Statistics:Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE @@ -3742,96 +3739,96 @@ Stage-0 limit:-1 Stage-1 Reducer 4 - File Output Operator [FS_28] + File Output Operator [FS_27] compressed:false Statistics:Num rows: 302 Data size: 53756 Basic stats: COMPLETE Column stats: NONE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Select Operator [SEL_27] + Select Operator [SEL_26] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 302 Data size: 53756 Basic stats: COMPLETE Column stats: NONE |<-Reducer 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_26] + Reduce Output Operator [RS_25] key expressions:_col0 (type: string) sort order:+ Statistics:Num rows: 302 Data size: 53756 Basic stats: COMPLETE Column stats: NONE value expressions:_col1 (type: string) - Select Operator [SEL_25] + Select Operator [SEL_24] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 302 Data size: 53756 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_31] + Filter Operator [FIL_30] predicate:_col3 is null (type: boolean) Statistics:Num rows: 302 Data size: 53756 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_36] + Merge Join Operator [MERGEJOIN_35] | condition map:[{"":"Left Outer Join0 to 1"}] | keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"} | outputColumnNames:["_col0","_col1","_col3"] | Statistics:Num rows: 605 Data size: 107690 Basic stats: COMPLETE Column stats: NONE |<-Map 7 [SIMPLE_EDGE] - | Reduce Output Operator [RS_22] + | Reduce Output Operator [RS_21] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_16] + | Select Operator [SEL_15] | outputColumnNames:["_col0"] | Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_34] + | Filter Operator [FIL_33] | predicate:(key > '2') (type: boolean) | Statistics:Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_14] + | TableScan [TS_13] | alias:src_cbo | Statistics:Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 2 [SIMPLE_EDGE] - Reduce Output Operator [RS_21] + Reduce Output Operator [RS_20] key expressions:_col0 (type: string) Map-reduce partition columns:_col0 (type: string) sort order:+ Statistics:Num rows: 550 Data size: 97900 Basic stats: COMPLETE Column stats: NONE value expressions:_col1 (type: string) - Merge Join Operator [MERGEJOIN_35] + Merge Join Operator [MERGEJOIN_34] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{} | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 550 Data size: 97900 Basic stats: COMPLETE Column stats: NONE |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_18] + | Reduce Output Operator [RS_17] | sort order: | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col0 (type: string), _col1 (type: string) - | Select Operator [SEL_2] + | Select Operator [SEL_1] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE | TableScan [TS_0] | alias:src_cbo | Statistics:Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 6 [SIMPLE_EDGE] - Reduce Output Operator [RS_19] + Reduce Output Operator [RS_18] sort order: Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator [SEL_11] + Select Operator [SEL_10] Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Filter Operator [FIL_32] + Filter Operator [FIL_31] predicate:(_col0 = 0) (type: boolean) Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_10] + Group By Operator [GBY_9] | aggregations:["count(VALUE._col0)"] | outputColumnNames:["_col0"] | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 5 [SIMPLE_EDGE] - Reduce Output Operator [RS_9] + Reduce Output Operator [RS_8] sort order: Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions:_col0 (type: bigint) - Group By Operator [GBY_8] + Group By Operator [GBY_7] aggregations:["count()"] outputColumnNames:["_col0"] Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_5] + Select Operator [SEL_4] Statistics:Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_33] + Filter Operator [FIL_32] predicate:((key > '2') and key is null) (type: boolean) Statistics:Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_3] + TableScan [TS_2] alias:src_cbo Statistics:Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE @@ -3863,87 +3860,87 @@ Stage-0 limit:-1 Stage-1 Reducer 3 - File Output Operator [FS_26] + File Output Operator [FS_25] compressed:false Statistics:Num rows: 15 Data size: 3507 Basic stats: COMPLETE Column stats: NONE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Select Operator [SEL_25] + Select Operator [SEL_24] outputColumnNames:["_col0","_col1","_col2"] Statistics:Num rows: 15 Data size: 3507 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_29] + Filter Operator [FIL_28] predicate:_col4 is null (type: boolean) Statistics:Num rows: 15 Data size: 3507 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_34] + Merge Join Operator [MERGEJOIN_33] | condition map:[{"":"Left Outer Join0 to 1"}] | keys:{"0":"_col0 (type: string), _col1 (type: string)","1":"_col0 (type: string), _col1 (type: string)"} | outputColumnNames:["_col0","_col1","_col2","_col4"] | Statistics:Num rows: 30 Data size: 7014 Basic stats: COMPLETE Column stats: NONE |<-Map 6 [SIMPLE_EDGE] - | Reduce Output Operator [RS_22] + | Reduce Output Operator [RS_21] | key expressions:_col0 (type: string), _col1 (type: string) | Map-reduce partition columns:_col0 (type: string), _col1 (type: string) | sort order:++ | Statistics:Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_16] + | Select Operator [SEL_15] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_32] + | Filter Operator [FIL_31] | predicate:(p_size < 10) (type: boolean) | Statistics:Num rows: 8 Data size: 1784 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_14] + | TableScan [TS_13] | alias:b | Statistics:Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 2 [SIMPLE_EDGE] - Reduce Output Operator [RS_21] + Reduce Output Operator [RS_20] key expressions:_col0 (type: string), _col1 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string) sort order:++ Statistics:Num rows: 28 Data size: 6377 Basic stats: COMPLETE Column stats: NONE value expressions:_col2 (type: int) - Merge Join Operator [MERGEJOIN_33] + Merge Join Operator [MERGEJOIN_32] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{} | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 28 Data size: 6377 Basic stats: COMPLETE Column stats: NONE |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_18] + | Reduce Output Operator [RS_17] | sort order: | Statistics:Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col0 (type: string), _col1 (type: string), _col2 (type: int) - | Select Operator [SEL_2] + | Select Operator [SEL_1] | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE | TableScan [TS_0] | alias:b | Statistics:Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 5 [SIMPLE_EDGE] - Reduce Output Operator [RS_19] + Reduce Output Operator [RS_18] sort order: Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator [SEL_11] + Select Operator [SEL_10] Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Filter Operator [FIL_30] + Filter Operator [FIL_29] predicate:(_col0 = 0) (type: boolean) Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_10] + Group By Operator [GBY_9] | aggregations:["count(VALUE._col0)"] | outputColumnNames:["_col0"] | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 4 [SIMPLE_EDGE] - Reduce Output Operator [RS_9] + Reduce Output Operator [RS_8] sort order: Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions:_col0 (type: bigint) - Group By Operator [GBY_8] + Group By Operator [GBY_7] aggregations:["count()"] outputColumnNames:["_col0"] Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_5] + Select Operator [SEL_4] Statistics:Num rows: 1 Data size: 223 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_31] + Filter Operator [FIL_30] predicate:((p_size < 10) and (p_name is null or p_mfgr is null)) (type: boolean) Statistics:Num rows: 1 Data size: 223 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_3] + TableScan [TS_2] alias:b Statistics:Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE @@ -3977,113 +3974,113 @@ Stage-0 limit:-1 Stage-1 Reducer 4 - File Output Operator [FS_37] + File Output Operator [FS_36] compressed:false Statistics:Num rows: 15 Data size: 1966 Basic stats: COMPLETE Column stats: NONE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Select Operator [SEL_36] + Select Operator [SEL_35] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 15 Data size: 1966 Basic stats: COMPLETE Column stats: NONE |<-Reducer 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_35] + Reduce Output Operator [RS_34] key expressions:_col0 (type: string) sort order:+ Statistics:Num rows: 15 Data size: 1966 Basic stats: COMPLETE Column stats: NONE value expressions:_col1 (type: int) - Select Operator [SEL_34] + Select Operator [SEL_33] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 15 Data size: 1966 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_40] + Filter Operator [FIL_39] predicate:_col3 is null (type: boolean) Statistics:Num rows: 15 Data size: 1966 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_47] + Merge Join Operator [MERGEJOIN_46] | condition map:[{"":"Left Outer Join0 to 1"}] | keys:{"0":"UDFToDouble(_col1) (type: double)","1":"_col0 (type: double)"} | outputColumnNames:["_col0","_col1","_col3"] | Statistics:Num rows: 30 Data size: 3932 Basic stats: COMPLETE Column stats: NONE |<-Reducer 2 [SIMPLE_EDGE] - | Reduce Output Operator [RS_30] + | Reduce Output Operator [RS_29] | key expressions:UDFToDouble(_col1) (type: double) | Map-reduce partition columns:UDFToDouble(_col1) (type: double) | sort order:+ | Statistics:Num rows: 28 Data size: 3575 Basic stats: COMPLETE Column stats: NONE | value expressions:_col0 (type: string), _col1 (type: int) - | Merge Join Operator [MERGEJOIN_46] + | Merge Join Operator [MERGEJOIN_45] | | condition map:[{"":"Inner Join 0 to 1"}] | | keys:{} | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 28 Data size: 3575 Basic stats: COMPLETE Column stats: NONE | |<-Map 1 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_27] + | | Reduce Output Operator [RS_26] | | sort order: | | Statistics:Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE | | value expressions:_col0 (type: string), _col1 (type: int) - | | Select Operator [SEL_2] + | | Select Operator [SEL_1] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE | | TableScan [TS_0] | | alias:part | | Statistics:Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE | |<-Reducer 6 [SIMPLE_EDGE] - | Reduce Output Operator [RS_28] + | Reduce Output Operator [RS_27] | sort order: | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - | Select Operator [SEL_16] + | Select Operator [SEL_15] | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - | Filter Operator [FIL_41] + | Filter Operator [FIL_40] | predicate:(_col0 = 0) (type: boolean) | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - | Group By Operator [GBY_15] + | Group By Operator [GBY_14] | aggregations:["count()"] | outputColumnNames:["_col0"] | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_9] + | Select Operator [SEL_8] | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_42] + | Filter Operator [FIL_41] | predicate:_col0 is null (type: boolean) | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - | Group By Operator [GBY_8] + | Group By Operator [GBY_7] | | aggregations:["avg(VALUE._col0)"] | | outputColumnNames:["_col0"] | | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE | |<-Map 5 [SIMPLE_EDGE] - | Reduce Output Operator [RS_7] + | Reduce Output Operator [RS_6] | sort order: | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE | value expressions:_col0 (type: struct) - | Group By Operator [GBY_6] + | Group By Operator [GBY_5] | aggregations:["avg(p_size)"] | outputColumnNames:["_col0"] | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - | Filter Operator [FIL_43] + | Filter Operator [FIL_42] | predicate:(p_size < 10) (type: boolean) | Statistics:Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_3] + | TableScan [TS_2] | alias:part | Statistics:Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 8 [SIMPLE_EDGE] - Reduce Output Operator [RS_31] + Reduce Output Operator [RS_30] key expressions:_col0 (type: double) Map-reduce partition columns:_col0 (type: double) sort order:+ Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_24] + Group By Operator [GBY_23] | aggregations:["avg(VALUE._col0)"] | outputColumnNames:["_col0"] | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 7 [SIMPLE_EDGE] - Reduce Output Operator [RS_23] + Reduce Output Operator [RS_22] sort order: Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE value expressions:_col0 (type: struct) - Group By Operator [GBY_22] + Group By Operator [GBY_21] aggregations:["avg(p_size)"] outputColumnNames:["_col0"] Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Filter Operator [FIL_45] + Filter Operator [FIL_44] predicate:(p_size < 10) (type: boolean) Statistics:Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_19] + TableScan [TS_18] alias:part Statistics:Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE @@ -4123,149 +4120,149 @@ Stage-0 limit:-1 Stage-1 Reducer 5 - File Output Operator [FS_40] + File Output Operator [FS_39] compressed:false Statistics:Num rows: 2 Data size: 256 Basic stats: COMPLETE Column stats: NONE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Select Operator [SEL_39] + Select Operator [SEL_38] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 2 Data size: 256 Basic stats: COMPLETE Column stats: NONE |<-Reducer 4 [SIMPLE_EDGE] - Reduce Output Operator [RS_38] + Reduce Output Operator [RS_37] key expressions:_col0 (type: string) sort order:+ Statistics:Num rows: 2 Data size: 256 Basic stats: COMPLETE Column stats: NONE value expressions:_col1 (type: double) - Select Operator [SEL_37] + Select Operator [SEL_36] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 2 Data size: 256 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_43] + Filter Operator [FIL_42] predicate:_col3 is null (type: boolean) Statistics:Num rows: 2 Data size: 256 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_49] + Merge Join Operator [MERGEJOIN_48] | condition map:[{"":"Left Outer Join0 to 1"}] | keys:{"0":"_col0 (type: string), _col1 (type: double)","1":"_col0 (type: string), _col1 (type: double)"} | outputColumnNames:["_col0","_col1","_col3"] | Statistics:Num rows: 5 Data size: 641 Basic stats: COMPLETE Column stats: NONE |<-Reducer 10 [SIMPLE_EDGE] - | Reduce Output Operator [RS_34] + | Reduce Output Operator [RS_33] | key expressions:_col0 (type: string), _col1 (type: double) | Map-reduce partition columns:_col0 (type: string), _col1 (type: double) | sort order:++ | Statistics:Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_28] + | Select Operator [SEL_27] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE - | Filter Operator [FIL_46] + | Filter Operator [FIL_45] | predicate:((_col2 - _col1) > 600.0) (type: boolean) | Statistics:Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE - | Group By Operator [GBY_26] + | Group By Operator [GBY_25] | | aggregations:["min(VALUE._col0)","max(VALUE._col1)"] | | keys:KEY._col0 (type: string) | | outputColumnNames:["_col0","_col1","_col2"] | | Statistics:Num rows: 5 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE | |<-Map 9 [SIMPLE_EDGE] - | Reduce Output Operator [RS_25] + | Reduce Output Operator [RS_24] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 5 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col1 (type: double), _col2 (type: double) - | Group By Operator [GBY_24] + | Group By Operator [GBY_23] | aggregations:["min(p_retailprice)","max(p_retailprice)"] | keys:p_mfgr (type: string) | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 5 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE - | TableScan [TS_22] + | TableScan [TS_21] | alias:b | Statistics:Num rows: 26 Data size: 2756 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_33] + Reduce Output Operator [RS_32] key expressions:_col0 (type: string), _col1 (type: double) Map-reduce partition columns:_col0 (type: string), _col1 (type: double) sort order:++ Statistics:Num rows: 5 Data size: 583 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_48] + Merge Join Operator [MERGEJOIN_47] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{} | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 5 Data size: 583 Basic stats: COMPLETE Column stats: NONE |<-Reducer 2 [SIMPLE_EDGE] - | Reduce Output Operator [RS_30] + | Reduce Output Operator [RS_29] | sort order: | Statistics:Num rows: 5 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col0 (type: string), _col1 (type: double) - | Group By Operator [GBY_5] + | Group By Operator [GBY_4] | | aggregations:["min(VALUE._col0)"] | | keys:KEY._col0 (type: string) | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 5 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE | |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_4] + | Reduce Output Operator [RS_3] | key expressions:_col0 (type: string) | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 5 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE | value expressions:_col1 (type: double) - | Group By Operator [GBY_3] + | Group By Operator [GBY_2] | aggregations:["min(p_retailprice)"] | keys:p_mfgr (type: string) | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 5 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE - | Select Operator [SEL_2] + | Select Operator [SEL_1] | outputColumnNames:["p_mfgr","p_retailprice"] | Statistics:Num rows: 26 Data size: 2756 Basic stats: COMPLETE Column stats: COMPLETE | TableScan [TS_0] | alias:b | Statistics:Num rows: 26 Data size: 2756 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 8 [SIMPLE_EDGE] - Reduce Output Operator [RS_31] + Reduce Output Operator [RS_30] sort order: Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator [SEL_19] + Select Operator [SEL_18] Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Filter Operator [FIL_44] + Filter Operator [FIL_43] predicate:(_col0 = 0) (type: boolean) Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_18] + Group By Operator [GBY_17] | aggregations:["count(VALUE._col0)"] | outputColumnNames:["_col0"] | Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE |<-Reducer 7 [SIMPLE_EDGE] - Reduce Output Operator [RS_17] + Reduce Output Operator [RS_16] sort order: Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions:_col0 (type: bigint) - Group By Operator [GBY_16] + Group By Operator [GBY_15] aggregations:["count()"] outputColumnNames:["_col0"] Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_13] + Select Operator [SEL_12] Statistics:Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator [FIL_45] + Filter Operator [FIL_44] predicate:((_col0 is null or _col1 is null) and ((_col2 - _col1) > 600.0)) (type: boolean) Statistics:Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator [GBY_11] + Group By Operator [GBY_10] | aggregations:["min(VALUE._col0)","max(VALUE._col1)"] | keys:KEY._col0 (type: string) | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 5 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE |<-Map 6 [SIMPLE_EDGE] - Reduce Output Operator [RS_10] + Reduce Output Operator [RS_9] key expressions:_col0 (type: string) Map-reduce partition columns:_col0 (type: string) sort order:+ Statistics:Num rows: 5 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE value expressions:_col1 (type: double), _col2 (type: double) - Group By Operator [GBY_9] + Group By Operator [GBY_8] aggregations:["min(p_retailprice)","max(p_retailprice)"] keys:p_mfgr (type: string) outputColumnNames:["_col0","_col1","_col2"] Statistics:Num rows: 5 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator [SEL_8] + Select Operator [SEL_7] outputColumnNames:["p_mfgr","p_retailprice"] Statistics:Num rows: 26 Data size: 2756 Basic stats: COMPLETE Column stats: COMPLETE - TableScan [TS_7] + TableScan [TS_6] alias:b Statistics:Num rows: 26 Data size: 2756 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/tez/explainuser_2.q.out b/ql/src/test/results/clientpositive/tez/explainuser_2.q.out index 8004641..e54dd0a 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_2.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_2.q.out @@ -198,8 +198,8 @@ Stage-0 Statistics:Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Merge Join Operator [MERGEJOIN_28] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"0":"_col3 (type: string)","1":"_col0 (type: string)"} - | outputColumnNames:["_col0","_col3","_col6"] + | keys:{"0":"_col1 (type: string)","1":"_col0 (type: string)"} + | outputColumnNames:["_col0","_col1","_col4"] | Statistics:Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE |<-Map 5 [SIMPLE_EDGE] | Reduce Output Operator [RS_15] @@ -219,15 +219,15 @@ Stage-0 | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE |<-Reducer 2 [SIMPLE_EDGE] Reduce Output Operator [RS_13] - key expressions:_col3 (type: string) - Map-reduce partition columns:_col3 (type: string) + key expressions:_col1 (type: string) + Map-reduce partition columns:_col1 (type: string) sort order:+ Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE value expressions:_col0 (type: string) Merge Join Operator [MERGEJOIN_27] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{"0":"_col0 (type: string)","1":"_col1 (type: string)"} - | outputColumnNames:["_col0","_col3"] + | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE |<-Map 1 [SIMPLE_EDGE] | Reduce Output Operator [RS_8] @@ -367,53 +367,53 @@ Stage-0 Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE value expressions:_col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint) Group By Operator [GBY_61] - aggregations:["count(_col13)","count(_col21)","count(_col3)"] - keys:_col2 (type: string), _col12 (type: string), _col20 (type: string) + aggregations:["count(_col9)","count(_col15)","count(_col3)"] + keys:_col2 (type: string), _col8 (type: string), _col14 (type: string) outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE Select Operator [SEL_60] - outputColumnNames:["_col2","_col12","_col20","_col13","_col21","_col3"] + outputColumnNames:["_col2","_col8","_col14","_col9","_col15","_col3"] Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE Merge Join Operator [MERGEJOIN_110] | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"0":"_col1 (type: string), _col3 (type: string)","1":"_col15 (type: string), _col17 (type: string)"} - | outputColumnNames:["_col2","_col3","_col12","_col13","_col20","_col21"] + | keys:{"0":"_col1 (type: string), _col3 (type: string)","1":"_col10 (type: string), _col12 (type: string)"} + | outputColumnNames:["_col2","_col3","_col8","_col9","_col14","_col15"] | Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE |<-Reducer 11 [SIMPLE_EDGE] | Reduce Output Operator [RS_58] - | key expressions:_col15 (type: string), _col17 (type: string) - | Map-reduce partition columns:_col15 (type: string), _col17 (type: string) + | key expressions:_col10 (type: string), _col12 (type: string) + | Map-reduce partition columns:_col10 (type: string), _col12 (type: string) | sort order:++ | Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE - | value expressions:_col6 (type: string), _col7 (type: string), _col14 (type: string) + | value expressions:_col3 (type: string), _col4 (type: string), _col9 (type: string) | Select Operator [SEL_49] - | outputColumnNames:["_col14","_col15","_col17","_col6","_col7"] + | outputColumnNames:["_col10","_col12","_col3","_col4","_col9"] | Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE | Merge Join Operator [MERGEJOIN_109] | | condition map:[{"":"Inner Join 0 to 1"}] - | | keys:{"0":"_col4 (type: string), _col6 (type: string)","1":"_col2 (type: string), _col4 (type: string)"} - | | outputColumnNames:["_col2","_col3","_col14","_col15","_col17"] + | | keys:{"0":"_col3 (type: string), _col5 (type: string)","1":"_col1 (type: string), _col3 (type: string)"} + | | outputColumnNames:["_col1","_col2","_col9","_col10","_col12"] | | Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE | |<-Reducer 10 [SIMPLE_EDGE] | | Reduce Output Operator [RS_45] - | | key expressions:_col4 (type: string), _col6 (type: string) - | | Map-reduce partition columns:_col4 (type: string), _col6 (type: string) + | | key expressions:_col3 (type: string), _col5 (type: string) + | | Map-reduce partition columns:_col3 (type: string), _col5 (type: string) | | sort order:++ | | Statistics:Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col2 (type: string), _col3 (type: string) + | | value expressions:_col1 (type: string), _col2 (type: string) | | Merge Join Operator [MERGEJOIN_107] | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"0":"_col3 (type: string)","1":"_col1 (type: string)"} - | | | outputColumnNames:["_col2","_col3","_col4","_col6"] + | | | keys:{"0":"_col2 (type: string)","1":"_col0 (type: string)"} + | | | outputColumnNames:["_col1","_col2","_col3","_col5"] | | | Statistics:Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE | | |<-Map 14 [SIMPLE_EDGE] | | | Reduce Output Operator [RS_42] - | | | key expressions:_col1 (type: string) - | | | Map-reduce partition columns:_col1 (type: string) + | | | key expressions:_col0 (type: string) + | | | Map-reduce partition columns:_col0 (type: string) | | | sort order:+ | | | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE | | | Select Operator [SEL_16] - | | | outputColumnNames:["_col1"] + | | | outputColumnNames:["_col0"] | | | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE | | | Filter Operator [FIL_101] | | | predicate:((key = 'src1key') and value is not null) (type: boolean) @@ -423,15 +423,15 @@ Stage-0 | | | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE | | |<-Reducer 9 [SIMPLE_EDGE] | | Reduce Output Operator [RS_40] - | | key expressions:_col3 (type: string) - | | Map-reduce partition columns:_col3 (type: string) + | | key expressions:_col2 (type: string) + | | Map-reduce partition columns:_col2 (type: string) | | sort order:+ | | Statistics:Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col2 (type: string), _col4 (type: string), _col6 (type: string) + | | value expressions:_col1 (type: string), _col3 (type: string), _col5 (type: string) | | Merge Join Operator [MERGEJOIN_106] | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"0":"_col2 (type: string)","1":"_col0 (type: string)"} - | | | outputColumnNames:["_col2","_col3","_col4","_col6"] + | | | keys:{"0":"_col1 (type: string)","1":"_col0 (type: string)"} + | | | outputColumnNames:["_col1","_col2","_col3","_col5"] | | | Statistics:Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE | | |<-Map 13 [SIMPLE_EDGE] | | | Reduce Output Operator [RS_37] @@ -450,15 +450,15 @@ Stage-0 | | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE | | |<-Reducer 8 [SIMPLE_EDGE] | | Reduce Output Operator [RS_35] - | | key expressions:_col2 (type: string) - | | Map-reduce partition columns:_col2 (type: string) + | | key expressions:_col1 (type: string) + | | Map-reduce partition columns:_col1 (type: string) | | sort order:+ | | Statistics:Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col3 (type: string), _col4 (type: string), _col6 (type: string) + | | value expressions:_col2 (type: string), _col3 (type: string), _col5 (type: string) | | Merge Join Operator [MERGEJOIN_105] | | | condition map:[{"":"Inner Join 0 to 1"}] - | | | keys:{"0":"_col1 (type: string)","1":"_col3 (type: string)"} - | | | outputColumnNames:["_col2","_col3","_col4","_col6"] + | | | keys:{"0":"_col0 (type: string)","1":"_col3 (type: string)"} + | | | outputColumnNames:["_col1","_col2","_col3","_col5"] | | | Statistics:Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE | | |<-Map 12 [SIMPLE_EDGE] | | | Reduce Output Operator [RS_32] @@ -478,12 +478,12 @@ Stage-0 | | | Statistics:Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE | | |<-Map 7 [SIMPLE_EDGE] | | Reduce Output Operator [RS_30] - | | key expressions:_col1 (type: string) - | | Map-reduce partition columns:_col1 (type: string) + | | key expressions:_col0 (type: string) + | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE | | Select Operator [SEL_7] - | | outputColumnNames:["_col1"] + | | outputColumnNames:["_col0"] | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE | | Filter Operator [FIL_98] | | predicate:((key = 'srcpartkey') and value is not null) (type: boolean) @@ -493,15 +493,15 @@ Stage-0 | | Statistics:Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE | |<-Reducer 16 [SIMPLE_EDGE] | Reduce Output Operator [RS_47] - | key expressions:_col2 (type: string), _col4 (type: string) - | Map-reduce partition columns:_col2 (type: string), _col4 (type: string) + | key expressions:_col1 (type: string), _col3 (type: string) + | Map-reduce partition columns:_col1 (type: string), _col3 (type: string) | sort order:++ | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE - | value expressions:_col3 (type: string), _col5 (type: string) + | value expressions:_col2 (type: string), _col4 (type: string) | Merge Join Operator [MERGEJOIN_108] | | condition map:[{"":"Inner Join 0 to 1"}] | | keys:{"0":"_col0 (type: string)","1":"_col0 (type: string)"} - | | outputColumnNames:["_col2","_col3","_col4","_col5"] + | | outputColumnNames:["_col1","_col2","_col3","_col4"] | | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE | |<-Map 15 [SIMPLE_EDGE] | | Reduce Output Operator [RS_24] @@ -509,9 +509,9 @@ Stage-0 | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + | | value expressions:_col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string) | | Select Operator [SEL_19] - | | outputColumnNames:["_col0","_col2","_col3","_col4","_col5"] + | | outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE | | Filter Operator [FIL_102] | | predicate:((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) @@ -1406,8 +1406,8 @@ Stage-0 Map Join Operator [MAPJOIN_28] | condition map:[{"":"Inner Join 0 to 1"}] | HybridGraceHashJoin:true - | keys:{"Map 1":"_col3 (type: string)","Map 3":"_col0 (type: string)"} - | outputColumnNames:["_col0","_col3","_col6"] + | keys:{"Map 1":"_col1 (type: string)","Map 3":"_col0 (type: string)"} + | outputColumnNames:["_col0","_col1","_col4"] | Statistics:Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE |<-Map 3 [BROADCAST_EDGE] | Reduce Output Operator [RS_15] @@ -1429,7 +1429,7 @@ Stage-0 | condition map:[{"":"Inner Join 0 to 1"}] | HybridGraceHashJoin:true | keys:{"Map 1":"_col0 (type: string)","Map 2":"_col1 (type: string)"} - | outputColumnNames:["_col0","_col3"] + | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE |<-Map 2 [BROADCAST_EDGE] | Reduce Output Operator [RS_10] @@ -1559,18 +1559,18 @@ Stage-0 Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE value expressions:_col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint) Group By Operator [GBY_61] - aggregations:["count(_col13)","count(_col21)","count(_col3)"] - keys:_col2 (type: string), _col12 (type: string), _col20 (type: string) + aggregations:["count(_col9)","count(_col15)","count(_col3)"] + keys:_col2 (type: string), _col8 (type: string), _col14 (type: string) outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE Select Operator [SEL_60] - outputColumnNames:["_col2","_col12","_col20","_col13","_col21","_col3"] + outputColumnNames:["_col2","_col8","_col14","_col9","_col15","_col3"] Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_110] | condition map:[{"":"Inner Join 0 to 1"}] | HybridGraceHashJoin:true - | keys:{"Map 2":"_col1 (type: string), _col3 (type: string)","Map 3":"_col15 (type: string), _col17 (type: string)"} - | outputColumnNames:["_col2","_col3","_col12","_col13","_col20","_col21"] + | keys:{"Map 2":"_col1 (type: string), _col3 (type: string)","Map 3":"_col10 (type: string), _col12 (type: string)"} + | outputColumnNames:["_col2","_col3","_col8","_col9","_col14","_col15"] | Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE |<-Map 2 [BROADCAST_EDGE] | Reduce Output Operator [RS_56] @@ -1611,26 +1611,26 @@ Stage-0 | alias:d1 | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE |<-Select Operator [SEL_49] - outputColumnNames:["_col14","_col15","_col17","_col6","_col7"] + outputColumnNames:["_col10","_col12","_col3","_col4","_col9"] Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_109] | condition map:[{"":"Inner Join 0 to 1"}] | HybridGraceHashJoin:true - | keys:{"Map 3":"_col4 (type: string), _col6 (type: string)","Map 10":"_col2 (type: string), _col4 (type: string)"} - | outputColumnNames:["_col2","_col3","_col14","_col15","_col17"] + | keys:{"Map 3":"_col3 (type: string), _col5 (type: string)","Map 10":"_col1 (type: string), _col3 (type: string)"} + | outputColumnNames:["_col1","_col2","_col9","_col10","_col12"] | Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE |<-Map 10 [BROADCAST_EDGE] | Reduce Output Operator [RS_47] - | key expressions:_col2 (type: string), _col4 (type: string) - | Map-reduce partition columns:_col2 (type: string), _col4 (type: string) + | key expressions:_col1 (type: string), _col3 (type: string) + | Map-reduce partition columns:_col1 (type: string), _col3 (type: string) | sort order:++ | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE - | value expressions:_col3 (type: string), _col5 (type: string) + | value expressions:_col2 (type: string), _col4 (type: string) | Map Join Operator [MAPJOIN_108] | | condition map:[{"":"Inner Join 0 to 1"}] | | HybridGraceHashJoin:true | | keys:{"Map 9":"_col0 (type: string)","Map 10":"_col0 (type: string)"} - | | outputColumnNames:["_col2","_col3","_col4","_col5"] + | | outputColumnNames:["_col1","_col2","_col3","_col4"] | | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE | |<-Map 9 [BROADCAST_EDGE] | | Reduce Output Operator [RS_24] @@ -1638,9 +1638,9 @@ Stage-0 | | Map-reduce partition columns:_col0 (type: string) | | sort order:+ | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE - | | value expressions:_col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + | | value expressions:_col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string) | | Select Operator [SEL_19] - | | outputColumnNames:["_col0","_col2","_col3","_col4","_col5"] + | | outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] | | Statistics:Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE | | Filter Operator [FIL_102] | | predicate:((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) @@ -1660,17 +1660,17 @@ Stage-0 |<-Map Join Operator [MAPJOIN_107] | condition map:[{"":"Inner Join 0 to 1"}] | HybridGraceHashJoin:true - | keys:{"Map 3":"_col3 (type: string)","Map 8":"_col1 (type: string)"} - | outputColumnNames:["_col2","_col3","_col4","_col6"] + | keys:{"Map 3":"_col2 (type: string)","Map 8":"_col0 (type: string)"} + | outputColumnNames:["_col1","_col2","_col3","_col5"] | Statistics:Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE |<-Map 8 [BROADCAST_EDGE] | Reduce Output Operator [RS_42] - | key expressions:_col1 (type: string) - | Map-reduce partition columns:_col1 (type: string) + | key expressions:_col0 (type: string) + | Map-reduce partition columns:_col0 (type: string) | sort order:+ | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE | Select Operator [SEL_16] - | outputColumnNames:["_col1"] + | outputColumnNames:["_col0"] | Statistics:Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE | Filter Operator [FIL_101] | predicate:((key = 'src1key') and value is not null) (type: boolean) @@ -1681,8 +1681,8 @@ Stage-0 |<-Map Join Operator [MAPJOIN_106] | condition map:[{"":"Inner Join 0 to 1"}] | HybridGraceHashJoin:true - | keys:{"Map 3":"_col2 (type: string)","Map 7":"_col0 (type: string)"} - | outputColumnNames:["_col2","_col3","_col4","_col6"] + | keys:{"Map 3":"_col1 (type: string)","Map 7":"_col0 (type: string)"} + | outputColumnNames:["_col1","_col2","_col3","_col5"] | Statistics:Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE |<-Map 7 [BROADCAST_EDGE] | Reduce Output Operator [RS_37] @@ -1702,8 +1702,8 @@ Stage-0 |<-Map Join Operator [MAPJOIN_105] | condition map:[{"":"Inner Join 0 to 1"}] | HybridGraceHashJoin:true - | keys:{"Map 3":"_col1 (type: string)","Map 6":"_col3 (type: string)"} - | outputColumnNames:["_col2","_col3","_col4","_col6"] + | keys:{"Map 3":"_col0 (type: string)","Map 6":"_col3 (type: string)"} + | outputColumnNames:["_col1","_col2","_col3","_col5"] | Statistics:Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE |<-Map 6 [BROADCAST_EDGE] | Reduce Output Operator [RS_32] @@ -1722,7 +1722,7 @@ Stage-0 | alias:ss | Statistics:Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE |<-Select Operator [SEL_7] - outputColumnNames:["_col1"] + outputColumnNames:["_col0"] Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator [FIL_98] predicate:((key = 'srcpartkey') and value is not null) (type: boolean) diff --git a/ql/src/test/results/clientpositive/tez/vector_decimal_round.q.out b/ql/src/test/results/clientpositive/tez/vector_decimal_round.q.out index 9a5d047..5bc04d7 100644 --- a/ql/src/test/results/clientpositive/tez/vector_decimal_round.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_decimal_round.q.out @@ -118,7 +118,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: round(_col0, (- 1)) (type: decimal(11,0)) + key expressions: round(_col0, -1) (type: decimal(11,0)) sort order: + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: decimal(10,0)) @@ -268,7 +268,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: round(_col0, (- 1)) (type: decimal(11,0)) + key expressions: round(_col0, -1) (type: decimal(11,0)) sort order: + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: decimal(10,0)) @@ -419,7 +419,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: round(_col0, (- 1)) (type: decimal(11,0)) + key expressions: round(_col0, -1) (type: decimal(11,0)) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: decimal(10,0)) diff --git a/ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out index 5e5d38e..38791dd 100644 --- a/ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out @@ -63,13 +63,13 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2, _col4 + outputColumnNames: _col2, _col3 input vertices: 1 Reducer 4 Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE HybridGraceHashJoin: true Select Operator - expressions: _col4 (type: int), _col2 (type: int) + expressions: _col3 (type: int), _col2 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -205,8 +205,8 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + 0 _col0 (type: int), 1 (type: int) + 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col1, _col2 input vertices: 1 Map 2 @@ -224,21 +224,21 @@ STAGE PLANS: alias: lineitem Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((l_shipmode = 'AIR') and (l_linenumber = 1)) and l_orderkey is not null) (type: boolean) + predicate: (((l_shipmode = 'AIR') and l_orderkey is not null) and l_linenumber is not null) (type: boolean) Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: l_orderkey (type: int) - outputColumnNames: _col0 + expressions: l_orderkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: int) + keys: _col0 (type: int), _col1 (type: int) mode: hash - outputColumnNames: _col0 + outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Map 3 Map Operator Tree: diff --git a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out index 9451b98..864d664 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out @@ -820,12 +820,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -843,7 +843,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -865,7 +865,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -959,12 +959,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Reducer 2 @@ -974,7 +974,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -1899,7 +1899,6 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -1923,42 +1922,42 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 4 Map Operator Tree: TableScan alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: ds + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: ds (type: string) - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: Merge Join Operator condition map: Inner Join 0 to 1 keys: - 0 - 1 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + 0 _col0 (type: string) + 1 '2008-04-08' (type: string) + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -1994,8 +1993,25 @@ STAGE PLANS: Select Operator Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: + key expressions: '2008-04-08' (type: string) + sort order: + + Map-reduce partition columns: '2008-04-08' (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '2008-04-08' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Dynamic Partitioning Event Operator + Target Input: srcpart + Partition key expr: ds + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Target column: ds + Target Vertex: Map 1 Stage: Stage-0 Fetch Operator @@ -2003,18 +2019,21 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -2073,12 +2092,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: vectorized Reducer 2 Reduce Operator Tree: @@ -2088,10 +2107,10 @@ STAGE PLANS: keys: 0 1 - outputColumnNames: _col0, _col1, _col2, _col4 + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((_col0 = _col2) or (_col1 = _col4)) (type: boolean) + predicate: ((_col0 = _col2) or (_col1 = _col3)) (type: boolean) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -2189,12 +2208,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -2212,7 +2231,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -2234,7 +2253,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -3923,7 +3942,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), _col1 (type: string) - 1 _col0 (type: string), _col2 (type: string) + 1 _col0 (type: string), _col1 (type: string) input vertices: 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -3948,12 +3967,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) @@ -3971,7 +3990,7 @@ STAGE PLANS: Target column: ds Target Vertex: Map 1 Select Operator - expressions: _col2 (type: string) + expressions: _col1 (type: string) outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Group By Operator @@ -4393,7 +4412,6 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: -- parent is reduce tasks EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY @@ -4416,33 +4434,33 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + filterExpr: ds is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + expressions: ds (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Map 2 Map Operator Tree: TableScan alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '2008-04-08' (type: string) - outputColumnNames: ds + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: ds (type: string) - mode: hash - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reducer 3 Execution mode: vectorized Reduce Operator Tree: @@ -4457,11 +4475,12 @@ STAGE PLANS: condition map: Inner Join 0 to 1 keys: - 0 - 1 + 0 _col0 (type: string) + 1 '2008-04-08' (type: string) input vertices: 0 Map 1 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + HybridGraceHashJoin: true Group By Operator aggregations: count() mode: hash @@ -4493,18 +4512,21 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### POSTHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 1000 PREHOOK: query: select count(*) from srcpart where ds = '2008-04-08' @@ -5344,7 +5366,7 @@ STAGE PLANS: Inner Join 0 to 1 keys: 0 _col0 (type: string), UDFToDouble(_col1) (type: double) - 1 _col0 (type: string), UDFToDouble(_col2) (type: double) + 1 _col0 (type: string), UDFToDouble(_col1) (type: double) input vertices: 1 Map 3 Statistics: Num rows: 1100 Data size: 103400 Basic stats: COMPLETE Column stats: NONE @@ -5369,12 +5391,12 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) - outputColumnNames: _col0, _col2 + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), UDFToDouble(_col2) (type: double) + key expressions: _col0 (type: string), UDFToDouble(_col1) (type: double) sort order: ++ - Map-reduce partition columns: _col0 (type: string), UDFToDouble(_col2) (type: double) + Map-reduce partition columns: _col0 (type: string), UDFToDouble(_col1) (type: double) Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) diff --git a/ql/src/test/results/clientpositive/udf1.q.out b/ql/src/test/results/clientpositive/udf1.q.out index dffbccf..b3b694b 100644 --- a/ql/src/test/results/clientpositive/udf1.q.out +++ b/ql/src/test/results/clientpositive/udf1.q.out @@ -137,26 +137,26 @@ POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT 'a' LIKE '%a%', 'b POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 -POSTHOOK: Lineage: dest1.c1 EXPRESSION [] -POSTHOOK: Lineage: dest1.c10 EXPRESSION [] -POSTHOOK: Lineage: dest1.c11 EXPRESSION [] -POSTHOOK: Lineage: dest1.c12 EXPRESSION [] -POSTHOOK: Lineage: dest1.c13 EXPRESSION [] -POSTHOOK: Lineage: dest1.c14 EXPRESSION [] -POSTHOOK: Lineage: dest1.c15 EXPRESSION [] -POSTHOOK: Lineage: dest1.c16 EXPRESSION [] -POSTHOOK: Lineage: dest1.c17 EXPRESSION [] -POSTHOOK: Lineage: dest1.c18 EXPRESSION [] -POSTHOOK: Lineage: dest1.c19 EXPRESSION [] -POSTHOOK: Lineage: dest1.c2 EXPRESSION [] -POSTHOOK: Lineage: dest1.c20 EXPRESSION [] -POSTHOOK: Lineage: dest1.c3 EXPRESSION [] -POSTHOOK: Lineage: dest1.c4 EXPRESSION [] -POSTHOOK: Lineage: dest1.c5 EXPRESSION [] -POSTHOOK: Lineage: dest1.c6 EXPRESSION [] -POSTHOOK: Lineage: dest1.c7 EXPRESSION [] -POSTHOOK: Lineage: dest1.c8 EXPRESSION [] -POSTHOOK: Lineage: dest1.c9 EXPRESSION [] +POSTHOOK: Lineage: dest1.c1 SIMPLE [] +POSTHOOK: Lineage: dest1.c10 SIMPLE [] +POSTHOOK: Lineage: dest1.c11 SIMPLE [] +POSTHOOK: Lineage: dest1.c12 SIMPLE [] +POSTHOOK: Lineage: dest1.c13 SIMPLE [] +POSTHOOK: Lineage: dest1.c14 SIMPLE [] +POSTHOOK: Lineage: dest1.c15 SIMPLE [] +POSTHOOK: Lineage: dest1.c16 SIMPLE [] +POSTHOOK: Lineage: dest1.c17 SIMPLE [] +POSTHOOK: Lineage: dest1.c18 SIMPLE [] +POSTHOOK: Lineage: dest1.c19 SIMPLE [] +POSTHOOK: Lineage: dest1.c2 SIMPLE [] +POSTHOOK: Lineage: dest1.c20 SIMPLE [] +POSTHOOK: Lineage: dest1.c3 SIMPLE [] +POSTHOOK: Lineage: dest1.c4 SIMPLE [] +POSTHOOK: Lineage: dest1.c5 SIMPLE [] +POSTHOOK: Lineage: dest1.c6 SIMPLE [] +POSTHOOK: Lineage: dest1.c7 SIMPLE [] +POSTHOOK: Lineage: dest1.c8 SIMPLE [] +POSTHOOK: Lineage: dest1.c9 SIMPLE [] PREHOOK: query: SELECT dest1.* FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 diff --git a/ql/src/test/results/clientpositive/udf_10_trims.q.out b/ql/src/test/results/clientpositive/udf_10_trims.q.out index 2f79723..3a5303a 100644 --- a/ql/src/test/results/clientpositive/udf_10_trims.q.out +++ b/ql/src/test/results/clientpositive/udf_10_trims.q.out @@ -117,4 +117,4 @@ WHERE src.key = 86 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 -POSTHOOK: Lineage: dest1.c1 EXPRESSION [] +POSTHOOK: Lineage: dest1.c1 SIMPLE [] diff --git a/ql/src/test/results/clientpositive/udf_concat_insert2.q.out b/ql/src/test/results/clientpositive/udf_concat_insert2.q.out index f1b70fe..d68bd76 100644 --- a/ql/src/test/results/clientpositive/udf_concat_insert2.q.out +++ b/ql/src/test/results/clientpositive/udf_concat_insert2.q.out @@ -16,7 +16,7 @@ INSERT OVERWRITE TABLE dest1 SELECT concat('1234', 'abc', 'extra argument'), src POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 -POSTHOOK: Lineage: dest1.key EXPRESSION [] +POSTHOOK: Lineage: dest1.key SIMPLE [] POSTHOOK: Lineage: dest1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT dest1.* FROM dest1 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/union_date_trim.q.out b/ql/src/test/results/clientpositive/union_date_trim.q.out index e2f5269..324e8b7 100644 --- a/ql/src/test/results/clientpositive/union_date_trim.q.out +++ b/ql/src/test/results/clientpositive/union_date_trim.q.out @@ -51,4 +51,4 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@testdate POSTHOOK: Output: default@testdate POSTHOOK: Lineage: testdate.dt EXPRESSION [(testdate)testdate.FieldSchema(name:dt, type:date, comment:null), ] -POSTHOOK: Lineage: testdate.id EXPRESSION [(testdate)testdate.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: testdate.id EXPRESSION [] diff --git a/ql/src/test/results/clientpositive/vector_decimal_round.q.out b/ql/src/test/results/clientpositive/vector_decimal_round.q.out index 25e5cfa..ec6226e 100644 --- a/ql/src/test/results/clientpositive/vector_decimal_round.q.out +++ b/ql/src/test/results/clientpositive/vector_decimal_round.q.out @@ -106,7 +106,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: round(_col0, (- 1)) (type: decimal(11,0)) + key expressions: round(_col0, -1) (type: decimal(11,0)) sort order: + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: decimal(10,0)) @@ -242,7 +242,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: round(_col0, (- 1)) (type: decimal(11,0)) + key expressions: round(_col0, -1) (type: decimal(11,0)) sort order: + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: decimal(10,0)) @@ -379,7 +379,7 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: round(_col0, (- 1)) (type: decimal(11,0)) + key expressions: round(_col0, -1) (type: decimal(11,0)) sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: decimal(10,0)) diff --git a/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out index 2973008..c7e338c 100644 --- a/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out +++ b/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out @@ -94,10 +94,10 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2, _col4 + outputColumnNames: _col2, _col3 Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: int), _col2 (type: int) + expressions: _col3 (type: int), _col2 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -134,10 +134,10 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2, _col4 + outputColumnNames: _col2, _col3 Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: int), _col2 (type: int) + expressions: _col3 (type: int), _col2 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -173,10 +173,10 @@ STAGE PLANS: keys: 0 _col1 (type: int) 1 _col0 (type: int) - outputColumnNames: _col2, _col4 + outputColumnNames: _col2, _col3 Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col4 (type: int), _col2 (type: int) + expressions: _col3 (type: int), _col2 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -457,30 +457,30 @@ STAGE PLANS: Stage: Stage-11 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_2:$hdt$_2:lineitem + $hdt$_2:lineitem Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_2:$hdt$_2:lineitem + $hdt$_2:lineitem TableScan alias: lineitem Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((l_shipmode = 'AIR') and (l_linenumber = 1)) and l_orderkey is not null) (type: boolean) + predicate: (((l_shipmode = 'AIR') and l_orderkey is not null) and l_linenumber is not null) (type: boolean) Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: l_orderkey (type: int) - outputColumnNames: _col0 + expressions: l_orderkey (type: int), l_linenumber (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: int) + keys: _col0 (type: int), _col1 (type: int) mode: hash - outputColumnNames: _col0 + outputColumnNames: _col0, _col1 Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + 0 _col0 (type: int), 1 (type: int) + 1 _col0 (type: int), _col1 (type: int) Stage: Stage-8 Map Reduce @@ -499,8 +499,8 @@ STAGE PLANS: condition map: Left Semi Join 0 to 1 keys: - 0 _col0 (type: int) - 1 _col0 (type: int) + 0 _col0 (type: int), 1 (type: int) + 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col1, _col2 Statistics: Num rows: 14 Data size: 1714 Basic stats: COMPLETE Column stats: NONE File Output Operator