diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index be38f38..7d6dbe8 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1656,9 +1656,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "If the skew information is correctly stored in the metadata, hive.optimize.skewjoin.compiletime\n" + "would change the query plan to take care of it, and hive.optimize.skewjoin will be a no-op."), - HIVE_SHARED_SCAN_OPTIMIZATION("hive.optimize.shared.scan", true, - "Whether to enable shared scan optimizer. The optimizer finds scan operator over the same table\n" + - "in the query plan and merges them if they meet some preconditions."), + HIVE_SHARED_WORK_OPTIMIZATION("hive.optimize.shared.scan", true, + "Whether to enable shared work optimizer. The optimizer finds scan operator over the same table\n" + + "and follow-up operators in the query plan and merges them if they meet some preconditions."), // CTE HIVE_CTE_MATERIALIZE_THRESHOLD("hive.optimize.cte.materialize.threshold", -1, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java index d5006bd..be0795d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java @@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.Lists; import com.google.common.collect.Multimap; public class OperatorUtils { @@ -381,6 +382,27 @@ public static void removeBranch(Operator op) { curr.removeChild(child); } + /** + * Remove operator from the tree, disconnecting it from its + * parents and children. + */ + public static void removeOperator(Operator op) { + if (op.getNumParent() != 0) { + List> allParent = + Lists.newArrayList(op.getParentOperators()); + for (Operator parentOp : allParent) { + parentOp.removeChild(op); + } + } + if (op.getNumChild() != 0) { + List> allChildren = + Lists.newArrayList(op.getChildOperators()); + for (Operator childOp : allChildren) { + childOp.removeParent(op); + } + } + } + public static String getOpNamePretty(Operator op) { if (op instanceof TableScanOperator) { return op.toString() + " (" + ((TableScanOperator) op).getConf().getAlias() + ")"; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java index a9099b8..bd2d73e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java @@ -21,43 +21,67 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Stack; -import com.google.common.base.Preconditions; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.exec.*; +import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.FilterOperator; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.GroupByOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.exec.SelectOperator; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; +import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.AcidUtils.Operation; -import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; -import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; -import org.apache.hadoop.hive.ql.lib.Dispatcher; -import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.spark.SparkPartitionPruningSinkDesc; -import org.apache.hadoop.hive.ql.parse.*; +import org.apache.hadoop.hive.ql.parse.GenTezUtils; +import org.apache.hadoop.hive.ql.parse.GenTezUtils.DynamicListContext; +import org.apache.hadoop.hive.ql.parse.GenTezUtils.DynamicPartitionPrunerContext; +import org.apache.hadoop.hive.ql.parse.OptimizeTezProcContext; +import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; +import org.apache.hadoop.hive.ql.parse.RuntimeValuesInfo; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.SemiJoinBranchInfo; +import org.apache.hadoop.hive.ql.parse.SemiJoinHint; import org.apache.hadoop.hive.ql.parse.spark.OptimizeSparkProcContext; -import org.apache.hadoop.hive.ql.plan.*; +import org.apache.hadoop.hive.ql.plan.AggregationDesc; +import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc; +import org.apache.hadoop.hive.ql.plan.DynamicValue; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicValueDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc; +import org.apache.hadoop.hive.ql.plan.GroupByDesc; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; +import org.apache.hadoop.hive.ql.plan.SelectDesc; +import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFBloomFilter.GenericUDAFBloomFilterEvaluator; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Preconditions; + /** * This optimization looks for expressions of the kind "x IN (RS[n])". If such * an expression made it to a table scan operator and x is a partition column we @@ -69,59 +93,6 @@ static final private Logger LOG = LoggerFactory.getLogger(DynamicPartitionPruningOptimization.class .getName()); - private static class DynamicPartitionPrunerProc implements NodeProcessor { - - /** - * process simply remembers all the dynamic partition pruning expressions - * found - */ - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - ExprNodeDynamicListDesc desc = (ExprNodeDynamicListDesc) nd; - DynamicPartitionPrunerContext context = (DynamicPartitionPrunerContext) procCtx; - - // Rule is searching for dynamic pruning expr. There's at least an IN - // expression wrapping it. - ExprNodeDesc parent = (ExprNodeDesc) stack.get(stack.size() - 2); - ExprNodeDesc grandParent = stack.size() >= 3 ? (ExprNodeDesc) stack.get(stack.size() - 3) : null; - - context.addDynamicList(desc, parent, grandParent, (ReduceSinkOperator) desc.getSource()); - - return context; - } - } - - private static class DynamicListContext { - public ExprNodeDynamicListDesc desc; - public ExprNodeDesc parent; - public ExprNodeDesc grandParent; - public ReduceSinkOperator generator; - - public DynamicListContext(ExprNodeDynamicListDesc desc, ExprNodeDesc parent, - ExprNodeDesc grandParent, ReduceSinkOperator generator) { - this.desc = desc; - this.parent = parent; - this.grandParent = grandParent; - this.generator = generator; - } - } - - private static class DynamicPartitionPrunerContext implements NodeProcessorCtx, - Iterable { - public List dynLists = new ArrayList(); - - public void addDynamicList(ExprNodeDynamicListDesc desc, ExprNodeDesc parent, - ExprNodeDesc grandParent, ReduceSinkOperator generator) { - dynLists.add(new DynamicListContext(desc, parent, grandParent, generator)); - } - - @Override - public Iterator iterator() { - return dynLists.iterator(); - } - } - @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { @@ -162,7 +133,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Obje // collect the dynamic pruning conditions removerContext.dynLists.clear(); - collectDynamicPruningConditions(desc.getPredicate(), removerContext); + GenTezUtils.collectDynamicPruningConditions(desc.getPredicate(), removerContext); if (ts == null) { // Replace the synthetic predicate with true and bail out @@ -402,7 +373,7 @@ private void cleanTableScanFilters(TableScanOperator ts) throws SemanticExceptio // collect the dynamic pruning conditions removerContext.dynLists.clear(); - collectDynamicPruningConditions(ts.getConf().getFilterExpr(), removerContext); + GenTezUtils.collectDynamicPruningConditions(ts.getConf().getFilterExpr(), removerContext); for (DynamicListContext ctx : removerContext) { // remove the condition by replacing it with "true" @@ -476,6 +447,7 @@ private void generateEventOperatorPlan(DynamicListContext ctx, ParseContext pars ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { DynamicPruningEventDesc eventDesc = new DynamicPruningEventDesc(); eventDesc.setTableScan(ts); + eventDesc.setGenerator(ctx.generator); eventDesc.setTable(PlanUtils.getReduceValueTableDesc(PlanUtils .getFieldSchemasFromColumnList(keyExprs, "key"))); eventDesc.setTargetColumnName(column); @@ -784,27 +756,4 @@ private void createFinalRsForSemiJoinOp( parseContext.getColExprToGBMap().put(key, gb); } - private Map collectDynamicPruningConditions(ExprNodeDesc pred, NodeProcessorCtx ctx) - throws SemanticException { - - // create a walker which walks the tree in a DFS manner while maintaining - // the operator stack. The dispatcher - // generates the plan from the operator tree - Map exprRules = new LinkedHashMap(); - exprRules.put(new RuleRegExp("R1", ExprNodeDynamicListDesc.class.getName() + "%"), - new DynamicPartitionPrunerProc()); - - // The dispatcher fires the processor corresponding to the closest matching - // rule and passes the context along - Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, ctx); - GraphWalker egw = new DefaultGraphWalker(disp); - - List startNodes = new ArrayList(); - startNodes.add(pred); - - HashMap outputMap = new HashMap(); - egw.startWalking(startNodes, outputMap); - return outputMap; - } - } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java index 1da9164..0373e53 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java @@ -25,6 +25,7 @@ import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator; import org.apache.hadoop.hive.ql.exec.CollectOperator; import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator; import org.apache.hadoop.hive.ql.exec.DemuxOperator; @@ -56,6 +57,9 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorLimitOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorSparkHashTableSinkOperator; +import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc; +import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc; +import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; @@ -93,13 +97,14 @@ new SparkHashTableSinkOperatorComparator()); comparatorMapping.put(LateralViewJoinOperator.class, new LateralViewJoinOperatorComparator()); comparatorMapping.put(VectorGroupByOperator.class, new VectorGroupByOperatorComparator()); - comparatorMapping.put(CommonMergeJoinOperator.class, new MapJoinOperatorComparator()); + comparatorMapping.put(CommonMergeJoinOperator.class, new CommonMergeJoinOperatorComparator()); comparatorMapping.put(VectorFilterOperator.class, new FilterOperatorComparator()); comparatorMapping.put(UDTFOperator.class, new UDTFOperatorComparator()); comparatorMapping.put(VectorSelectOperator.class, new VectorSelectOperatorComparator()); comparatorMapping.put(VectorLimitOperator.class, new LimitOperatorComparator()); comparatorMapping.put(ScriptOperator.class, new ScriptOperatorComparator()); comparatorMapping.put(TemporaryHashSinkOperator.class, new HashTableSinkOperatorComparator()); + comparatorMapping.put(AppMasterEventOperator.class, new AppMasterEventOperatorComparator()); // these operators does not have state, so they always equal with the same kind. comparatorMapping.put(UnionOperator.class, new AlwaysTrueOperatorComparator()); comparatorMapping.put(ForwardOperator.class, new AlwaysTrueOperatorComparator()); @@ -364,7 +369,33 @@ public boolean equals(MapJoinOperator op1, MapJoinOperator op2) { compareObject(desc1.getKeysString(), desc2.getKeysString()) && desc1.getPosBigTable() == desc2.getPosBigTable() && desc1.isBucketMapJoin() == desc2.isBucketMapJoin() && + compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) && + compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) && + compareObject(desc1.getCondsList(), desc2.getCondsList()) && + desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() && + compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) { + return true; + } else { + return false; + } + } + } + + static class CommonMergeJoinOperatorComparator implements OperatorComparator { + + @Override + public boolean equals(CommonMergeJoinOperator op1, CommonMergeJoinOperator op2) { + Preconditions.checkNotNull(op1); + Preconditions.checkNotNull(op2); + CommonMergeJoinDesc desc1 = op1.getConf(); + CommonMergeJoinDesc desc2 = op2.getConf(); + + if (compareObject(desc1.getParentToInput(), desc2.getParentToInput()) && + compareString(desc1.getKeyCountsExplainDesc(), desc2.getKeyCountsExplainDesc()) && compareObject(desc1.getKeysString(), desc2.getKeysString()) && + desc1.getPosBigTable() == desc2.getPosBigTable() && + desc1.isBucketMapJoin() == desc2.isBucketMapJoin() && + desc1.getNumBuckets() == desc2.getNumBuckets() && compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) && compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) && compareObject(desc1.getCondsList(), desc2.getCondsList()) && @@ -516,6 +547,37 @@ public boolean equals(UDTFOperator op1, UDTFOperator op2) { } } + static class AppMasterEventOperatorComparator implements OperatorComparator { + + @Override + public boolean equals(AppMasterEventOperator op1, AppMasterEventOperator op2) { + Preconditions.checkNotNull(op1); + Preconditions.checkNotNull(op2); + AppMasterEventDesc op1Conf = op1.getConf(); + AppMasterEventDesc op2Conf = op2.getConf(); + + if (compareString(op1Conf.getInputName(), op2Conf.getInputName()) && + compareString(op1Conf.getVertexName(), op2Conf.getVertexName()) && + compareObject(op1Conf.getTable(), op2Conf.getTable())) { + if (op1Conf instanceof DynamicPruningEventDesc && op2Conf instanceof DynamicPruningEventDesc) { + DynamicPruningEventDesc op1DPPConf = (DynamicPruningEventDesc) op1Conf; + DynamicPruningEventDesc op2DPPConf = (DynamicPruningEventDesc) op2Conf; + if (compareString(op1DPPConf.getTargetColumnName(), op2DPPConf.getTargetColumnName()) && + compareString(op1DPPConf.getTargetColumnType(), op2DPPConf.getTargetColumnType()) && + compareString(op1DPPConf.getPartKeyString(), op2DPPConf.getPartKeyString())) { + return true; + } + return false; + } else if (op1Conf instanceof DynamicPruningEventDesc || op2Conf instanceof DynamicPruningEventDesc) { + return false; + } + return true; + } else { + return false; + } + } + } + static boolean compareString(String first, String second) { return compareObject(first, second); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedScanOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedScanOptimizer.java deleted file mode 100644 index e31119f..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedScanOptimizer.java +++ /dev/null @@ -1,626 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.optimizer; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator; -import org.apache.hadoop.hive.ql.exec.DummyStoreOperator; -import org.apache.hadoop.hive.ql.exec.FilterOperator; -import org.apache.hadoop.hive.ql.exec.MapJoinOperator; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.OperatorFactory; -import org.apache.hadoop.hive.ql.exec.OperatorUtils; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.RowSchema; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.UDFArgumentException; -import org.apache.hadoop.hive.ql.exec.UnionOperator; -import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.SemiJoinBranchInfo; -import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.stats.StatsUtils; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; - -/** - * Shared scan optimizer. This rule finds scan operator over the same table - * in the query plan and merges them if they meet some preconditions. - * - * TS TS TS - * | | -> / \ - * Op Op Op Op - * - *

Currently it only works with the Tez execution engine. - */ -public class SharedScanOptimizer extends Transform { - - private final static Logger LOG = LoggerFactory.getLogger(SharedScanOptimizer.class); - - @Override - public ParseContext transform(ParseContext pctx) throws SemanticException { - - final Map topOps = pctx.getTopOps(); - if (topOps.size() < 2) { - // Nothing to do, bail out - return pctx; - } - - // Cache to use during optimization - SharedScanOptimizerCache optimizerCache = new SharedScanOptimizerCache(); - - // We will not apply this optimization on some table scan operators. - Set excludeTableScanOps = gatherNotValidTableScanOps(pctx, optimizerCache); - LOG.debug("Exclude TableScan ops: {}", excludeTableScanOps); - - // Map of dbName.TblName -> Pair(tableAlias, TSOperator) - Multimap> tableNameToOps = splitTableScanOpsByTable(pctx); - - // We enforce a certain order when we do the reutilization. - // In particular, we use size of table x number of reads to - // rank the tables. - List> sortedTables = rankTablesByAccumulatedSize(pctx, excludeTableScanOps); - LOG.debug("Sorted tables by size: {}", sortedTables); - - // Execute optimization - Multimap existingOps = ArrayListMultimap.create(); - Set entriesToRemove = new HashSet<>(); - for (Entry tablePair : sortedTables) { - for (Entry tableScanOpPair : tableNameToOps.get(tablePair.getKey())) { - TableScanOperator tsOp = tableScanOpPair.getValue(); - if (excludeTableScanOps.contains(tsOp)) { - // Skip operator, currently we do not merge - continue; - } - String tableName = tablePair.getKey(); - Collection prevTsOps = existingOps.get(tableName); - if (!prevTsOps.isEmpty()) { - for (TableScanOperator prevTsOp : prevTsOps) { - - // First we check if the two table scan operators can actually be merged - // If schemas do not match, we currently do not merge - List prevTsOpNeededColumns = prevTsOp.getNeededColumns(); - List tsOpNeededColumns = tsOp.getNeededColumns(); - if (prevTsOpNeededColumns.size() != tsOpNeededColumns.size()) { - // Skip - continue; - } - boolean notEqual = false; - for (int i = 0; i < prevTsOpNeededColumns.size(); i++) { - if (!prevTsOpNeededColumns.get(i).equals(tsOpNeededColumns.get(i))) { - notEqual = true; - break; - } - } - if (notEqual) { - // Skip - continue; - } - // If row limit does not match, we currently do not merge - if (prevTsOp.getConf().getRowLimit() != tsOp.getConf().getRowLimit()) { - // Skip - continue; - } - // If partitions do not match, we currently do not merge - PrunedPartitionList prevTsOpPPList = pctx.getPrunedPartitions(prevTsOp); - PrunedPartitionList tsOpPPList = pctx.getPrunedPartitions(tsOp); - if (prevTsOpPPList.hasUnknownPartitions() - || tsOpPPList.hasUnknownPartitions() - || !prevTsOpPPList.getPartitions().equals(tsOpPPList.getPartitions())) { - // Skip - continue; - } - - // It seems these two operators can be merged. - // Check that plan meets some preconditions before doing it. - // In particular, in the presence of map joins in the upstream plan: - // - we cannot exceed the noconditional task size, and - // - if we already merged the big table, we cannot merge the broadcast - // tables. - if (!validPreConditions(pctx, optimizerCache, prevTsOp, tsOp)) { - // Skip - LOG.debug("{} does not meet preconditions", tsOp); - continue; - } - - // We can merge - ExprNodeGenericFuncDesc exprNode = null; - if (prevTsOp.getConf().getFilterExpr() != null) { - // Push filter on top of children - pushFilterToTopOfTableScan(optimizerCache, prevTsOp); - // Clone to push to table scan - exprNode = (ExprNodeGenericFuncDesc) prevTsOp.getConf().getFilterExpr(); - } - if (tsOp.getConf().getFilterExpr() != null) { - // Push filter on top - pushFilterToTopOfTableScan(optimizerCache, tsOp); - ExprNodeGenericFuncDesc tsExprNode = tsOp.getConf().getFilterExpr(); - if (exprNode != null && !exprNode.isSame(tsExprNode)) { - // We merge filters from previous scan by ORing with filters from current scan - if (exprNode.getGenericUDF() instanceof GenericUDFOPOr) { - List newChildren = new ArrayList<>(exprNode.getChildren().size() + 1); - for (ExprNodeDesc childExprNode : exprNode.getChildren()) { - if (childExprNode.isSame(tsExprNode)) { - // We do not need to do anything, it is in the OR expression - break; - } - newChildren.add(childExprNode); - } - if (exprNode.getChildren().size() == newChildren.size()) { - newChildren.add(tsExprNode); - exprNode = ExprNodeGenericFuncDesc.newInstance( - new GenericUDFOPOr(), - newChildren); - } - } else { - exprNode = ExprNodeGenericFuncDesc.newInstance( - new GenericUDFOPOr(), - Arrays.asList(exprNode, tsExprNode)); - } - } - } - // Replace filter - prevTsOp.getConf().setFilterExpr(exprNode); - // Replace table scan operator - List> allChildren = - Lists.newArrayList(tsOp.getChildOperators()); - for (Operator op : allChildren) { - tsOp.getChildOperators().remove(op); - op.replaceParent(tsOp, prevTsOp); - prevTsOp.getChildOperators().add(op); - } - entriesToRemove.add(tableScanOpPair.getKey()); - // Remove and combine - optimizerCache.removeOpAndCombineWork(tsOp, prevTsOp); - - LOG.debug("Merged {} into {}", tsOp, prevTsOp); - - break; - } - if (!entriesToRemove.contains(tableScanOpPair.getKey())) { - existingOps.put(tableName, tsOp); - } - } else { - // Add to existing ops - existingOps.put(tableName, tsOp); - } - } - } - // Remove unused operators - for (String key : entriesToRemove) { - topOps.remove(key); - } - - return pctx; - } - - private static Set gatherNotValidTableScanOps( - ParseContext pctx, SharedScanOptimizerCache optimizerCache) throws SemanticException { - // Find TS operators with partition pruning enabled in plan - // because these TS may potentially read different data for - // different pipeline. - // These can be: - // 1) TS with DPP. - // TODO: Check if dynamic filters are identical and do not add. - // 2) TS with semijoin DPP. - // TODO: Check for dynamic filters. - Set notValidTableScanOps = new HashSet<>(); - // 1) TS with DPP. - Map topOps = pctx.getTopOps(); - Collection> tableScanOps = - Lists.>newArrayList(topOps.values()); - Set s = - OperatorUtils.findOperators(tableScanOps, AppMasterEventOperator.class); - for (AppMasterEventOperator a : s) { - if (a.getConf() instanceof DynamicPruningEventDesc) { - DynamicPruningEventDesc dped = (DynamicPruningEventDesc) a.getConf(); - notValidTableScanOps.add(dped.getTableScan()); - optimizerCache.tableScanToDPPSource.put(dped.getTableScan(), a); - } - } - // 2) TS with semijoin DPP. - for (Entry e - : pctx.getRsToSemiJoinBranchInfo().entrySet()) { - notValidTableScanOps.add(e.getValue().getTsOp()); - optimizerCache.tableScanToDPPSource.put(e.getValue().getTsOp(), e.getKey()); - } - return notValidTableScanOps; - } - - private static Multimap> splitTableScanOpsByTable( - ParseContext pctx) { - Multimap> tableNameToOps = ArrayListMultimap.create(); - for (Entry e : pctx.getTopOps().entrySet()) { - TableScanOperator tsOp = e.getValue(); - tableNameToOps.put( - tsOp.getConf().getTableMetadata().getDbName() + "." - + tsOp.getConf().getTableMetadata().getTableName(), e); - } - return tableNameToOps; - } - - private static List> rankTablesByAccumulatedSize(ParseContext pctx, - Set excludeTables) { - Map tableToTotalSize = new HashMap<>(); - for (Entry e : pctx.getTopOps().entrySet()) { - TableScanOperator tsOp = e.getValue(); - if (excludeTables.contains(tsOp)) { - // Skip operator, currently we do not merge - continue; - } - String tableName = tsOp.getConf().getTableMetadata().getDbName() + "." - + tsOp.getConf().getTableMetadata().getTableName(); - long tableSize = tsOp.getStatistics() != null ? - tsOp.getStatistics().getDataSize() : 0L; - Long totalSize = tableToTotalSize.get(tableName); - if (totalSize != null) { - tableToTotalSize.put(tableName, - StatsUtils.safeAdd(totalSize, tableSize)); - } else { - tableToTotalSize.put(tableName, tableSize); - } - } - List> sortedTables = - new LinkedList<>(tableToTotalSize.entrySet()); - Collections.sort(sortedTables, Collections.reverseOrder( - new Comparator>() { - public int compare(Map.Entry o1, Map.Entry o2) { - return (o1.getValue()).compareTo(o2.getValue()); - } - })); - return sortedTables; - } - - private static boolean validPreConditions(ParseContext pctx, SharedScanOptimizerCache optimizerCache, - TableScanOperator prevTsOp, TableScanOperator tsOp) { - // 1) The set of operators in the works of the TS operators need to meet - // some requirements. In particular: - // 1.1. None of the works that contain the TS operators can contain a Union - // operator. This is not supported yet as we might end up with cycles in - // the Tez DAG. - // 1.2. There cannot be more than one DummyStore operator in the new resulting - // work when the TS operators are merged. This is due to an assumption in - // MergeJoinProc that needs to be further explored. - // If any of these conditions are not met, we cannot merge. - // TODO: Extend rule so it can be applied for these cases. - final Set> workOps1 = findWorkOperators(optimizerCache, prevTsOp); - final Set> workOps2 = findWorkOperators(optimizerCache, tsOp); - boolean foundDummyStoreOp = false; - for (Operator op : workOps1) { - if (op instanceof UnionOperator) { - // We cannot merge (1.1) - return false; - } - if (op instanceof DummyStoreOperator) { - foundDummyStoreOp = true; - } - } - for (Operator op : workOps2) { - if (op instanceof UnionOperator) { - // We cannot merge (1.1) - return false; - } - if (foundDummyStoreOp && op instanceof DummyStoreOperator) { - // We cannot merge (1.2) - return false; - } - } - // 2) We check whether output works when we merge the operators will collide. - // - // Work1 Work2 (merge TS in W1 & W2) Work1 - // \ / -> | | X - // Work3 Work3 - // - // If we do, we cannot merge. The reason is that Tez currently does - // not support parallel edges, i.e., multiple edges from same work x - // into same work y. - final Set> outputWorksOps1 = findChildWorkOperators(pctx, optimizerCache, prevTsOp); - final Set> outputWorksOps2 = findChildWorkOperators(pctx, optimizerCache, tsOp); - if (!Collections.disjoint(outputWorksOps1, outputWorksOps2)) { - // We cannot merge - return false; - } - // 3) We check whether we will end up with same operators inputing on same work. - // - // Work1 (merge TS in W2 & W3) Work1 - // / \ -> | | X - // Work2 Work3 Work2 - // - // If we do, we cannot merge. The reason is the same as above, currently - // Tez currently does not support parallel edges. - final Set> inputWorksOps1 = findParentWorkOperators(pctx, optimizerCache, prevTsOp); - final Set> inputWorksOps2 = findParentWorkOperators(pctx, optimizerCache, tsOp); - if (!Collections.disjoint(inputWorksOps1, inputWorksOps2)) { - // We cannot merge - return false; - } - // 4) We check whether one of the operators is part of a work that is an input for - // the work of the other operator. - // - // Work1 (merge TS in W1 & W3) Work1 - // | -> | X - // Work2 Work2 - // | | - // Work3 Work1 - // - // If we do, we cannot merge, as we would end up with a cycle in the DAG. - final Set> descendantWorksOps1 = - findDescendantWorkOperators(pctx, optimizerCache, prevTsOp); - final Set> descendantWorksOps2 = - findDescendantWorkOperators(pctx, optimizerCache, tsOp); - if (!Collections.disjoint(descendantWorksOps1, workOps2) - || !Collections.disjoint(workOps1, descendantWorksOps2)) { - return false; - } - // 5) We check whether merging the works would cause the size of - // the data in memory grow too large. - // TODO: Currently ignores GBY and PTF which may also buffer data in memory. - final Set> newWorkOps = workOps1; - newWorkOps.addAll(workOps2); - long dataSize = 0L; - for (Operator op : newWorkOps) { - if (op instanceof MapJoinOperator) { - MapJoinOperator mop = (MapJoinOperator) op; - dataSize = StatsUtils.safeAdd(dataSize, mop.getConf().getInMemoryDataSize()); - if (dataSize > mop.getConf().getMemoryMonitorInfo().getAdjustedNoConditionalTaskSize()) { - // Size surpasses limit, we cannot convert - LOG.debug("accumulated data size: {} / max size: {}", - dataSize, mop.getConf().getMemoryMonitorInfo().getAdjustedNoConditionalTaskSize()); - return false; - } - } - } - return true; - } - - private static Set> findParentWorkOperators(ParseContext pctx, - SharedScanOptimizerCache optimizerCache, Operator start) { - // Find operators in work - Set> workOps = findWorkOperators(optimizerCache, start); - // Gather input works operators - Set> set = new HashSet>(); - for (Operator op : workOps) { - if (op.getParentOperators() != null) { - for (Operator parent : op.getParentOperators()) { - if (parent instanceof ReduceSinkOperator) { - set.addAll(findWorkOperators(optimizerCache, parent)); - } - } - } else if (op instanceof TableScanOperator) { - // Check for DPP and semijoin DPP - for (Operator parent : optimizerCache.tableScanToDPPSource.get((TableScanOperator) op)) { - set.addAll(findWorkOperators(optimizerCache, parent)); - } - } - } - return set; - } - - private static Set> findChildWorkOperators(ParseContext pctx, - SharedScanOptimizerCache optimizerCache, Operator start) { - // Find operators in work - Set> workOps = findWorkOperators(optimizerCache, start); - // Gather output works operators - Set> set = new HashSet>(); - for (Operator op : workOps) { - if (op instanceof ReduceSinkOperator) { - if (op.getChildOperators() != null) { - // All children of RS are descendants - for (Operator child : op.getChildOperators()) { - set.addAll(findWorkOperators(optimizerCache, child)); - } - } - // Semijoin DPP work is considered a child because work needs - // to finish for it to execute - SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op); - if (sjbi != null) { - set.addAll(findWorkOperators(optimizerCache, sjbi.getTsOp())); - } - } else if(op.getConf() instanceof DynamicPruningEventDesc) { - // DPP work is considered a child because work needs - // to finish for it to execute - set.addAll(findWorkOperators( - optimizerCache, ((DynamicPruningEventDesc) op.getConf()).getTableScan())); - } - } - return set; - } - - private static Set> findDescendantWorkOperators(ParseContext pctx, - SharedScanOptimizerCache optimizerCache, Operator start) { - // Find operators in work - Set> workOps = findWorkOperators(optimizerCache, start); - // Gather output works operators - Set> result = new HashSet>(); - Set> set; - while (!workOps.isEmpty()) { - set = new HashSet>(); - for (Operator op : workOps) { - if (op instanceof ReduceSinkOperator) { - if (op.getChildOperators() != null) { - // All children of RS are descendants - for (Operator child : op.getChildOperators()) { - set.addAll(findWorkOperators(optimizerCache, child)); - } - } - // Semijoin DPP work is considered a descendant because work needs - // to finish for it to execute - SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op); - if (sjbi != null) { - set.addAll(findWorkOperators(optimizerCache, sjbi.getTsOp())); - } - } else if(op.getConf() instanceof DynamicPruningEventDesc) { - // DPP work is considered a descendant because work needs - // to finish for it to execute - set.addAll(findWorkOperators( - optimizerCache, ((DynamicPruningEventDesc) op.getConf()).getTableScan())); - } - } - workOps = set; - result.addAll(set); - } - return result; - } - - // Stores result in cache - private static Set> findWorkOperators( - SharedScanOptimizerCache optimizerCache, Operator start) { - Set> c = optimizerCache.operatorToWorkOperators.get(start); - if (!c.isEmpty()) { - return c; - } - c = findWorkOperators(start, new HashSet>()); - for (Operator op : c) { - optimizerCache.operatorToWorkOperators.putAll(op, c); - } - return c; - } - - private static Set> findWorkOperators(Operator start, Set> found) { - found.add(start); - if (start.getParentOperators() != null) { - for (Operator parent : start.getParentOperators()) { - if (parent instanceof ReduceSinkOperator) { - continue; - } - if (!found.contains(parent)) { - findWorkOperators(parent, found); - } - } - } - if (start instanceof ReduceSinkOperator) { - return found; - } - if (start.getChildOperators() != null) { - for (Operator child : start.getChildOperators()) { - if (!found.contains(child)) { - findWorkOperators(child, found); - } - } - } - return found; - } - - private static void pushFilterToTopOfTableScan( - SharedScanOptimizerCache optimizerCache, TableScanOperator tsOp) - throws UDFArgumentException { - ExprNodeGenericFuncDesc tableScanExprNode = tsOp.getConf().getFilterExpr(); - List> allChildren = - Lists.newArrayList(tsOp.getChildOperators()); - for (Operator op : allChildren) { - if (op instanceof FilterOperator) { - FilterOperator filterOp = (FilterOperator) op; - ExprNodeDesc filterExprNode = filterOp.getConf().getPredicate(); - if (tableScanExprNode.isSame(filterExprNode)) { - // We do not need to do anything - return; - } - if (tableScanExprNode.getGenericUDF() instanceof GenericUDFOPOr) { - for (ExprNodeDesc childExprNode : tableScanExprNode.getChildren()) { - if (childExprNode.isSame(filterExprNode)) { - // We do not need to do anything, it is in the OR expression - // so probably we pushed previously - return; - } - } - } - ExprNodeGenericFuncDesc newPred = ExprNodeGenericFuncDesc.newInstance( - new GenericUDFOPAnd(), - Arrays.asList(tableScanExprNode.clone(), filterExprNode)); - filterOp.getConf().setPredicate(newPred); - } else { - Operator newOp = OperatorFactory.get(tsOp.getCompilationOpContext(), - new FilterDesc(tableScanExprNode.clone(), false), - new RowSchema(tsOp.getSchema().getSignature())); - tsOp.replaceChild(op, newOp); - newOp.getParentOperators().add(tsOp); - op.replaceParent(tsOp, newOp); - newOp.getChildOperators().add(op); - // Add to cache (same group as tsOp) - optimizerCache.putIfWorkExists(newOp, tsOp); - } - } - } - - /** Cache to accelerate optimization */ - private static class SharedScanOptimizerCache { - // Operators that belong to each work - final HashMultimap, Operator> operatorToWorkOperators = - HashMultimap., Operator>create(); - // Table scan operators to DPP sources - final Multimap> tableScanToDPPSource = - HashMultimap.>create(); - - // Add new operator to cache work group of existing operator (if group exists) - void putIfWorkExists(Operator opToAdd, Operator existingOp) { - List> c = ImmutableList.copyOf(operatorToWorkOperators.get(existingOp)); - if (!c.isEmpty()) { - for (Operator op : c) { - operatorToWorkOperators.get(op).add(opToAdd); - } - operatorToWorkOperators.putAll(opToAdd, c); - operatorToWorkOperators.put(opToAdd, opToAdd); - } - } - - // Remove operator and combine - void removeOpAndCombineWork(Operator opToRemove, Operator replacementOp) { - Set> s = operatorToWorkOperators.get(opToRemove); - s.remove(opToRemove); - List> c1 = ImmutableList.copyOf(s); - List> c2 = ImmutableList.copyOf(operatorToWorkOperators.get(replacementOp)); - if (!c1.isEmpty() && !c2.isEmpty()) { - for (Operator op1 : c1) { - operatorToWorkOperators.remove(op1, opToRemove); // Remove operator - operatorToWorkOperators.putAll(op1, c2); // Add ops of new collection - } - operatorToWorkOperators.removeAll(opToRemove); // Remove entry for operator - for (Operator op2 : c2) { - operatorToWorkOperators.putAll(op2, c1); // Add ops to existing collection - } - } - } - } - -} diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java new file mode 100644 index 0000000..8070c2a --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java @@ -0,0 +1,1176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator; +import org.apache.hadoop.hive.ql.exec.DummyStoreOperator; +import org.apache.hadoop.hive.ql.exec.FilterOperator; +import org.apache.hadoop.hive.ql.exec.MapJoinOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.exec.OperatorUtils; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UnionOperator; +import org.apache.hadoop.hive.ql.parse.GenTezUtils; +import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.SemiJoinBranchInfo; +import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicListDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicValueDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; +import org.apache.hadoop.hive.ql.plan.TableScanDesc; +import org.apache.hadoop.hive.ql.stats.StatsUtils; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInBloomFilter; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multiset; +import com.google.common.collect.TreeMultiset; + +/** + * Shared computation optimizer. + * + *

Originally, this rule would find scan operators over the same table + * in the query plan and merge them if they met some preconditions. + * + * TS TS TS + * | | -> / \ + * Op Op Op Op + * + *

Now the rule has been extended to find opportunities to other operators + * downstream, not only a single table scan. + * + * TS1 TS2 TS1 TS2 TS1 TS2 + * | | | | | | + * | RS | RS | RS + * \ / \ / -> \ / + * MapJoin MapJoin MapJoin + * | | / \ + * Op Op Op Op + * + *

A limitation in the current implementation is that the optimizer does not + * go beyond a work boundary. + * + *

The optimization only works with the Tez execution engine. + */ +public class SharedWorkOptimizer extends Transform { + + private final static Logger LOG = LoggerFactory.getLogger(SharedWorkOptimizer.class); + + @Override + public ParseContext transform(ParseContext pctx) throws SemanticException { + + final Map topOps = pctx.getTopOps(); + if (topOps.size() < 2) { + // Nothing to do, bail out + return pctx; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Before SharedWorkOptimizer:\n" + Operator.toString(pctx.getTopOps().values())); + } + + // Cache to use during optimization + SharedWorkOptimizerCache optimizerCache = new SharedWorkOptimizerCache(); + + // Gather information about the DPP table scans and store it in the cache + gatherDPPTableScanOps(pctx, optimizerCache); + + // Map of dbName.TblName -> TSOperator + Multimap tableNameToOps = splitTableScanOpsByTable(pctx); + + // We enforce a certain order when we do the reutilization. + // In particular, we use size of table x number of reads to + // rank the tables. + List> sortedTables = rankTablesByAccumulatedSize(pctx); + LOG.debug("Sorted tables by size: {}", sortedTables); + + // Execute optimization + Multimap existingOps = ArrayListMultimap.create(); + Set> removedOps = new HashSet<>(); + for (Entry tablePair : sortedTables) { + String tableName = tablePair.getKey(); + for (TableScanOperator discardableTsOp : tableNameToOps.get(tableName)) { + if (removedOps.contains(discardableTsOp)) { + LOG.debug("Skip {} as it has been already removed", discardableTsOp); + continue; + } + Collection prevTsOps = existingOps.get(tableName); + for (TableScanOperator retainableTsOp : prevTsOps) { + if (removedOps.contains(retainableTsOp)) { + LOG.debug("Skip {} as it has been already removed", retainableTsOp); + continue; + } + + // First we quickly check if the two table scan operators can actually be merged + boolean mergeable = areMergeable(pctx, optimizerCache, retainableTsOp, discardableTsOp); + if (!mergeable) { + // Skip + LOG.debug("{} and {} cannot be merged", retainableTsOp, discardableTsOp); + continue; + } + + // Secondly, we extract information about the part of the tree that can be merged + // as well as some structural information (memory consumption) that needs to be + // used to determined whether the merge can happen + SharedResult sr = extractSharedOptimizationInfo( + pctx, optimizerCache, retainableTsOp, discardableTsOp); + + // It seems these two operators can be merged. + // Check that plan meets some preconditions before doing it. + // In particular, in the presence of map joins in the upstream plan: + // - we cannot exceed the noconditional task size, and + // - if we already merged the big table, we cannot merge the broadcast + // tables. + if (!validPreConditions(pctx, optimizerCache, sr)) { + // Skip + LOG.debug("{} and {} do not meet preconditions", retainableTsOp, discardableTsOp); + continue; + } + + // We can merge + if (sr.retainableOps.size() > 1) { + // More than TS operator + Operator lastRetainableOp = sr.retainableOps.get(sr.retainableOps.size() - 1); + Operator lastDiscardableOp = sr.discardableOps.get(sr.discardableOps.size() - 1); + if (lastDiscardableOp.getNumChild() != 0) { + List> allChildren = + Lists.newArrayList(lastDiscardableOp.getChildOperators()); + for (Operator op : allChildren) { + lastDiscardableOp.getChildOperators().remove(op); + op.replaceParent(lastDiscardableOp, lastRetainableOp); + lastRetainableOp.getChildOperators().add(op); + } + } + + LOG.debug("Merging subtree starting at {} into subtree starting at {}", discardableTsOp, retainableTsOp); + } else { + // Only TS operator + ExprNodeGenericFuncDesc exprNode = null; + if (retainableTsOp.getConf().getFilterExpr() != null) { + // Push filter on top of children + pushFilterToTopOfTableScan(optimizerCache, retainableTsOp); + // Clone to push to table scan + exprNode = (ExprNodeGenericFuncDesc) retainableTsOp.getConf().getFilterExpr(); + } + if (discardableTsOp.getConf().getFilterExpr() != null) { + // Push filter on top + pushFilterToTopOfTableScan(optimizerCache, discardableTsOp); + ExprNodeGenericFuncDesc tsExprNode = discardableTsOp.getConf().getFilterExpr(); + if (exprNode != null && !exprNode.isSame(tsExprNode)) { + // We merge filters from previous scan by ORing with filters from current scan + if (exprNode.getGenericUDF() instanceof GenericUDFOPOr) { + List newChildren = new ArrayList<>(exprNode.getChildren().size() + 1); + for (ExprNodeDesc childExprNode : exprNode.getChildren()) { + if (childExprNode.isSame(tsExprNode)) { + // We do not need to do anything, it is in the OR expression + break; + } + newChildren.add(childExprNode); + } + if (exprNode.getChildren().size() == newChildren.size()) { + newChildren.add(tsExprNode); + exprNode = ExprNodeGenericFuncDesc.newInstance( + new GenericUDFOPOr(), + newChildren); + } + } else { + exprNode = ExprNodeGenericFuncDesc.newInstance( + new GenericUDFOPOr(), + Arrays.asList(exprNode, tsExprNode)); + } + } + } + // Replace filter + retainableTsOp.getConf().setFilterExpr(exprNode); + // Replace table scan operator + List> allChildren = + Lists.newArrayList(discardableTsOp.getChildOperators()); + for (Operator op : allChildren) { + discardableTsOp.getChildOperators().remove(op); + op.replaceParent(discardableTsOp, retainableTsOp); + retainableTsOp.getChildOperators().add(op); + } + + LOG.debug("Merging {} into {}", discardableTsOp, retainableTsOp); + } + + // First we remove the input operators of the expression that + // we are going to eliminate + for (Operator op : sr.discardableInputOps) { + OperatorUtils.removeOperator(op); + optimizerCache.removeOp(op); + removedOps.add(op); + // Remove DPP predicates + if (op instanceof ReduceSinkOperator) { + SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op); + if (sjbi != null && !sr.discardableOps.contains(sjbi.getTsOp()) && + !sr.discardableInputOps.contains(sjbi.getTsOp())) { + GenTezUtils.removeSemiJoinOperator( + pctx, (ReduceSinkOperator) op, sjbi.getTsOp()); + } + } else if (op instanceof AppMasterEventOperator) { + DynamicPruningEventDesc dped = (DynamicPruningEventDesc) op.getConf(); + if (!sr.discardableOps.contains(dped.getTableScan()) && + !sr.discardableInputOps.contains(dped.getTableScan())) { + GenTezUtils.removeSemiJoinOperator( + pctx, (AppMasterEventOperator) op, dped.getTableScan()); + } + } + LOG.debug("Input operator removed: {}", op); + } + // Then we merge the operators of the works we are going to merge + optimizerCache.removeOpAndCombineWork(discardableTsOp, retainableTsOp); + removedOps.add(discardableTsOp); + // Finally we remove the expression from the tree + for (Operator op : sr.discardableOps) { + OperatorUtils.removeOperator(op); + optimizerCache.removeOp(op); + removedOps.add(op); + if (sr.discardableOps.size() == 1) { + // If there is a single discardable operator, it is a TableScanOperator + // and it means that we have merged filter expressions for it. Thus, we + // might need to remove DPP predicates from the retainable TableScanOperator + Collection> c = + optimizerCache.tableScanToDPPSource.get((TableScanOperator) op); + for (Operator dppSource : c) { + if (dppSource instanceof ReduceSinkOperator) { + GenTezUtils.removeSemiJoinOperator(pctx, + (ReduceSinkOperator) dppSource, + (TableScanOperator) sr.retainableOps.get(0)); + } else if (dppSource instanceof AppMasterEventOperator) { + GenTezUtils.removeSemiJoinOperator(pctx, + (AppMasterEventOperator) dppSource, + (TableScanOperator) sr.retainableOps.get(0)); + } + } + } + LOG.debug("Operator removed: {}", op); + } + + break; + } + + if (removedOps.contains(discardableTsOp)) { + // This operator has been removed, remove it from the list of existing operators + existingOps.remove(tableName, discardableTsOp); + } else { + // This operator has not been removed, include it in the list of existing operators + existingOps.put(tableName, discardableTsOp); + } + } + } + + // Remove unused table scan operators + Iterator> it = topOps.entrySet().iterator(); + while (it.hasNext()) { + Entry e = it.next(); + if (e.getValue().getNumChild() == 0) { + it.remove(); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("After SharedWorkOptimizer:\n" + Operator.toString(pctx.getTopOps().values())); + } + + return pctx; + } + + /** + * This method gathers the TS operators with DPP from the context and + * stores them into the input optimization cache. + */ + private static void gatherDPPTableScanOps( + ParseContext pctx, SharedWorkOptimizerCache optimizerCache) throws SemanticException { + // Find TS operators with partition pruning enabled in plan + // because these TS may potentially read different data for + // different pipeline. + // These can be: + // 1) TS with DPP. + // 2) TS with semijoin DPP. + Map topOps = pctx.getTopOps(); + Collection> tableScanOps = + Lists.>newArrayList(topOps.values()); + Set s = + OperatorUtils.findOperators(tableScanOps, AppMasterEventOperator.class); + for (AppMasterEventOperator a : s) { + if (a.getConf() instanceof DynamicPruningEventDesc) { + DynamicPruningEventDesc dped = (DynamicPruningEventDesc) a.getConf(); + optimizerCache.tableScanToDPPSource.put(dped.getTableScan(), a); + } + } + for (Entry e + : pctx.getRsToSemiJoinBranchInfo().entrySet()) { + optimizerCache.tableScanToDPPSource.put(e.getValue().getTsOp(), e.getKey()); + } + LOG.debug("DPP information stored in the cache: {}", optimizerCache.tableScanToDPPSource); + } + + private static Multimap splitTableScanOpsByTable( + ParseContext pctx) { + Multimap tableNameToOps = ArrayListMultimap.create(); + for (Entry e : pctx.getTopOps().entrySet()) { + TableScanOperator tsOp = e.getValue(); + tableNameToOps.put( + tsOp.getConf().getTableMetadata().getDbName() + "." + + tsOp.getConf().getTableMetadata().getTableName(), tsOp); + } + return tableNameToOps; + } + + private static List> rankTablesByAccumulatedSize(ParseContext pctx) { + Map tableToTotalSize = new HashMap<>(); + for (Entry e : pctx.getTopOps().entrySet()) { + TableScanOperator tsOp = e.getValue(); + String tableName = tsOp.getConf().getTableMetadata().getDbName() + "." + + tsOp.getConf().getTableMetadata().getTableName(); + long tableSize = tsOp.getStatistics() != null ? + tsOp.getStatistics().getDataSize() : 0L; + Long totalSize = tableToTotalSize.get(tableName); + if (totalSize != null) { + tableToTotalSize.put(tableName, + StatsUtils.safeAdd(totalSize, tableSize)); + } else { + tableToTotalSize.put(tableName, tableSize); + } + } + List> sortedTables = + new LinkedList<>(tableToTotalSize.entrySet()); + Collections.sort(sortedTables, Collections.reverseOrder( + new Comparator>() { + public int compare(Map.Entry o1, Map.Entry o2) { + return (o1.getValue()).compareTo(o2.getValue()); + } + })); + return sortedTables; + } + + private static boolean areMergeable(ParseContext pctx, SharedWorkOptimizerCache optimizerCache, + TableScanOperator tsOp1, TableScanOperator tsOp2) throws SemanticException { + // First we check if the two table scan operators can actually be merged + // If schemas do not match, we currently do not merge + List prevTsOpNeededColumns = tsOp1.getNeededColumns(); + List tsOpNeededColumns = tsOp2.getNeededColumns(); + if (prevTsOpNeededColumns.size() != tsOpNeededColumns.size()) { + return false; + } + boolean notEqual = false; + for (int i = 0; i < prevTsOpNeededColumns.size(); i++) { + if (!prevTsOpNeededColumns.get(i).equals(tsOpNeededColumns.get(i))) { + notEqual = true; + break; + } + } + if (notEqual) { + return false; + } + // If row limit does not match, we currently do not merge + if (tsOp1.getConf().getRowLimit() != tsOp2.getConf().getRowLimit()) { + return false; + } + // If partitions do not match, we currently do not merge + PrunedPartitionList prevTsOpPPList = pctx.getPrunedPartitions(tsOp1); + PrunedPartitionList tsOpPPList = pctx.getPrunedPartitions(tsOp2); + if (!prevTsOpPPList.getPartitions().equals(tsOpPPList.getPartitions())) { + return false; + } + // If is a DPP, check if actually it refers to same target, column, etc. + // Further, the DPP value needs to be generated from same subtree + List> dppsOp1 = new ArrayList<>(optimizerCache.tableScanToDPPSource.get(tsOp1)); + List> dppsOp2 = new ArrayList<>(optimizerCache.tableScanToDPPSource.get(tsOp2)); + if (dppsOp1.isEmpty() && dppsOp2.isEmpty()) { + return true; + } + for (int i = 0; i < dppsOp1.size(); i++) { + Operator op = dppsOp1.get(i); + if (op instanceof ReduceSinkOperator) { + Set> ascendants = + findAscendantWorkOperators(pctx, optimizerCache, op); + if (ascendants.contains(tsOp2)) { + dppsOp1.remove(i); + i--; + } + } + } + for (int i = 0; i < dppsOp2.size(); i++) { + Operator op = dppsOp2.get(i); + if (op instanceof ReduceSinkOperator) { + Set> ascendants = + findAscendantWorkOperators(pctx, optimizerCache, op); + if (ascendants.contains(tsOp1)) { + dppsOp2.remove(i); + i--; + } + } + } + if (dppsOp1.size() != dppsOp2.size()) { + // Only first or second operator contains DPP pruning + return false; + } + // Check if DPP branches are equal + for (int i = 0; i < dppsOp1.size(); i++) { + Operator dppOp1 = dppsOp1.get(i); + BitSet bs = new BitSet(); + for (int j = 0; j < dppsOp2.size(); j++) { + if (!bs.get(j)) { + // If not visited yet + Operator dppOp2 = dppsOp2.get(j); + if (compareAndGatherOps(pctx, dppOp1, dppOp2) != null) { + // The DPP operator/branch are equal + bs.set(j); + break; + } + } + } + if (bs.cardinality() == i) { + return false; + } + } + return true; + } + + private static SharedResult extractSharedOptimizationInfo(ParseContext pctx, + SharedWorkOptimizerCache optimizerCache, + TableScanOperator retainableTsOp, + TableScanOperator discardableTsOp) throws SemanticException { + Set> retainableOps = new LinkedHashSet<>(); + Set> discardableOps = new LinkedHashSet<>(); + Set> discardableInputOps = new HashSet<>(); + long dataSize = 0l; + long maxDataSize = 0l; + + retainableOps.add(retainableTsOp); + discardableOps.add(discardableTsOp); + Operator equalOp1 = retainableTsOp; + Operator equalOp2 = discardableTsOp; + if (equalOp1.getNumChild() > 1 || equalOp2.getNumChild() > 1) { + // TODO: Support checking multiple child operators to merge further. + discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps)); + return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize); + } + Operator currentOp1 = retainableTsOp.getChildOperators().get(0); + Operator currentOp2 = discardableTsOp.getChildOperators().get(0); + + // Special treatment for Filter operator that ignores the DPP predicates + if (currentOp1 instanceof FilterOperator && currentOp2 instanceof FilterOperator) { + boolean equalFilters = false; + FilterDesc op1Conf = ((FilterOperator) currentOp1).getConf(); + FilterDesc op2Conf = ((FilterOperator) currentOp2).getConf(); + + if (op1Conf.getIsSamplingPred() == op2Conf.getIsSamplingPred() && + StringUtils.equals(op1Conf.getSampleDescExpr(), op2Conf.getSampleDescExpr())) { + Multiset conjsOp1String = extractConjsIgnoringDPPPreds(op1Conf.getPredicate()); + Multiset conjsOp2String = extractConjsIgnoringDPPPreds(op2Conf.getPredicate()); + if (conjsOp1String.equals(conjsOp2String)) { + equalFilters = true; + } + } + + if (equalFilters) { + equalOp1 = currentOp1; + equalOp2 = currentOp2; + retainableOps.add(equalOp1); + discardableOps.add(equalOp2); + if (currentOp1.getChildOperators().size() > 1 || + currentOp2.getChildOperators().size() > 1) { + // TODO: Support checking multiple child operators to merge further. + discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableInputOps)); + discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps)); + discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps, discardableInputOps)); + return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize); + } + currentOp1 = currentOp1.getChildOperators().get(0); + currentOp2 = currentOp2.getChildOperators().get(0); + } else { + // Bail out + discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableInputOps)); + discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps)); + discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps, discardableInputOps)); + return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize); + } + } + + // Try to merge rest of operators + while (!(currentOp1 instanceof ReduceSinkOperator)) { + // Check whether current operators are equal + if (!compareOperator(pctx, currentOp1, currentOp2)) { + // If they are not equal, we could zip up till here + break; + } + if (currentOp1.getParentOperators().size() != + currentOp2.getParentOperators().size()) { + // If they are not equal, we could zip up till here + break; + } + if (currentOp1.getParentOperators().size() > 1) { + List> discardableOpsForCurrentOp = new ArrayList<>(); + int idx = 0; + for (; idx < currentOp1.getParentOperators().size(); idx++) { + Operator parentOp1 = currentOp1.getParentOperators().get(idx); + Operator parentOp2 = currentOp2.getParentOperators().get(idx); + if (parentOp1 == equalOp1 && parentOp2 == equalOp2) { + continue; + } + if ((parentOp1 == equalOp1 && parentOp2 != equalOp2) || + (parentOp1 != equalOp1 && parentOp2 == equalOp2)) { + // Input operator is not in the same position + break; + } + // Compare input + List> removeOpsForCurrentInput = compareAndGatherOps(pctx, parentOp1, parentOp2); + if (removeOpsForCurrentInput == null) { + // Inputs are not the same, bail out + break; + } + // Add inputs to ops to remove + discardableOpsForCurrentOp.addAll(removeOpsForCurrentInput); + } + if (idx != currentOp1.getParentOperators().size()) { + // If inputs are not equal, we could zip up till here + break; + } + discardableInputOps.addAll(discardableOpsForCurrentOp); + } + + equalOp1 = currentOp1; + equalOp2 = currentOp2; + retainableOps.add(equalOp1); + discardableOps.add(equalOp2); + if (equalOp1 instanceof MapJoinOperator) { + MapJoinOperator mop = (MapJoinOperator) equalOp1; + dataSize = StatsUtils.safeAdd(dataSize, mop.getConf().getInMemoryDataSize()); + maxDataSize = mop.getConf().getMemoryMonitorInfo().getAdjustedNoConditionalTaskSize(); + } + if (currentOp1.getChildOperators().size() > 1 || + currentOp2.getChildOperators().size() > 1) { + // TODO: Support checking multiple child operators to merge further. + break; + } + // Update for next iteration + currentOp1 = currentOp1.getChildOperators().get(0); + currentOp2 = currentOp2.getChildOperators().get(0); + } + + // Add the rest to the memory consumption + Set> opsWork1 = findWorkOperators(optimizerCache, currentOp1); + for (Operator op : opsWork1) { + if (op instanceof MapJoinOperator && !retainableOps.contains(op)) { + MapJoinOperator mop = (MapJoinOperator) op; + dataSize = StatsUtils.safeAdd(dataSize, mop.getConf().getInMemoryDataSize()); + maxDataSize = mop.getConf().getMemoryMonitorInfo().getAdjustedNoConditionalTaskSize(); + } + } + Set> opsWork2 = findWorkOperators(optimizerCache, currentOp2); + for (Operator op : opsWork2) { + if (op instanceof MapJoinOperator && !discardableOps.contains(op)) { + MapJoinOperator mop = (MapJoinOperator) op; + dataSize = StatsUtils.safeAdd(dataSize, mop.getConf().getInMemoryDataSize()); + maxDataSize = mop.getConf().getMemoryMonitorInfo().getAdjustedNoConditionalTaskSize(); + } + } + + discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableInputOps)); + discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps)); + discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps, discardableInputOps)); + return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize); + } + + private static Multiset extractConjsIgnoringDPPPreds(ExprNodeDesc predicate) { + List conjsOp = ExprNodeDescUtils.split(predicate); + Multiset conjsOpString = TreeMultiset.create(); + for (int i = 0; i < conjsOp.size(); i++) { + if (conjsOp.get(i) instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc func = (ExprNodeGenericFuncDesc) conjsOp.get(i); + if (GenericUDFInBloomFilter.class == func.getGenericUDF().getClass()) { + continue; + } else if (GenericUDFBetween.class == func.getGenericUDF().getClass() && + (func.getChildren().get(2) instanceof ExprNodeDynamicValueDesc || + func.getChildren().get(3) instanceof ExprNodeDynamicValueDesc)) { + continue; + } + } else if(conjsOp.get(i) instanceof ExprNodeDynamicListDesc) { + continue; + } + conjsOpString.add(conjsOp.get(i).toString()); + } + return conjsOpString; + } + + private static Set> gatherDPPBranchOps(ParseContext pctx, + SharedWorkOptimizerCache optimizerCache, Set> ops) { + Set> dppBranches = new HashSet<>(); + for (Operator op : ops) { + if (op instanceof TableScanOperator) { + Collection> c = optimizerCache.tableScanToDPPSource.get((TableScanOperator) op); + for (Operator dppSource : c) { + // Remove the branches + Operator currentOp = dppSource; + while (currentOp.getNumChild() <= 1) { + dppBranches.add(currentOp); + currentOp = currentOp.getParentOperators().get(0); + } + } + } + } + return dppBranches; + } + + private static Set> gatherDPPBranchOps(ParseContext pctx, + SharedWorkOptimizerCache optimizerCache, Set> ops, + Set> discardedOps) { + Set> dppBranches = new HashSet<>(); + for (Operator op : ops) { + if (op instanceof TableScanOperator) { + Collection> c = optimizerCache.tableScanToDPPSource.get((TableScanOperator) op); + for (Operator dppSource : c) { + Set> ascendants = + findAscendantWorkOperators(pctx, optimizerCache, dppSource); + if (!Collections.disjoint(ascendants, discardedOps)) { + // Remove branch + Operator currentOp = dppSource; + while (currentOp.getNumChild() <= 1) { + dppBranches.add(currentOp); + currentOp = currentOp.getParentOperators().get(0); + } + } + } + } + } + return dppBranches; + } + + private static List> compareAndGatherOps(ParseContext pctx, + Operator op1, Operator op2) throws SemanticException { + List> result = new ArrayList<>(); + boolean mergeable = compareAndGatherOps(pctx, op1, op2, result, true); + if (!mergeable) { + return null; + } + return result; + } + + private static boolean compareAndGatherOps(ParseContext pctx, Operator op1, Operator op2, + List> result, boolean gather) throws SemanticException { + if (!compareOperator(pctx, op1, op2)) { + LOG.debug("Operators not equal: {} and {}", op1, op2); + return false; + } + + if (gather) { + result.add(op2); + } + + List> op1ParentOperators = op1.getParentOperators(); + List> op2ParentOperators = op2.getParentOperators(); + if (op1ParentOperators != null && op2ParentOperators != null) { + if (op1ParentOperators.size() != op2ParentOperators.size()) { + return false; + } + for (int i = 0; i < op1ParentOperators.size(); i++) { + Operator op1ParentOp = op1ParentOperators.get(i); + Operator op2ParentOp = op2ParentOperators.get(i); + boolean mergeable; + if (gather && op2ParentOp.getChildOperators().size() < 2) { + mergeable = compareAndGatherOps(pctx, op1ParentOp, op2ParentOp, result, true); + } else { + mergeable = compareAndGatherOps(pctx, op1ParentOp, op2ParentOp, result, false); + } + if (!mergeable) { + return false; + } + } + } else if (op1ParentOperators != null || op2ParentOperators != null) { + return false; + } + + return true; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private static boolean compareOperator(ParseContext pctx, Operator op1, Operator op2) + throws SemanticException { + if (!op1.getClass().getName().equals(op2.getClass().getName())) { + return false; + } + + // We handle ReduceSinkOperator here as we can safely ignore table alias + // and the current comparator implementation does not. + // We can ignore table alias since when we compare ReduceSinkOperator, all + // its ancestors need to match (down to table scan), thus we make sure that + // both plans are the same. + if (op1 instanceof ReduceSinkOperator) { + ReduceSinkDesc op1Conf = ((ReduceSinkOperator) op1).getConf(); + ReduceSinkDesc op2Conf = ((ReduceSinkOperator) op2).getConf(); + + if (StringUtils.equals(op1Conf.getKeyColString(), op2Conf.getKeyColString()) && + StringUtils.equals(op1Conf.getValueColsString(), op2Conf.getValueColsString()) && + StringUtils.equals(op1Conf.getParitionColsString(), op2Conf.getParitionColsString()) && + op1Conf.getTag() == op2Conf.getTag() && + StringUtils.equals(op1Conf.getOrder(), op2Conf.getOrder()) && + op1Conf.getTopN() == op2Conf.getTopN() && + op1Conf.isAutoParallel() == op2Conf.isAutoParallel()) { + return true; + } else { + return false; + } + } + + // We handle TableScanOperator here as we can safely ignore table alias + // and the current comparator implementation does not. + if (op1 instanceof TableScanOperator) { + TableScanOperator tsOp1 = (TableScanOperator) op1; + TableScanOperator tsOp2 = (TableScanOperator) op2; + TableScanDesc op1Conf = tsOp1.getConf(); + TableScanDesc op2Conf = tsOp2.getConf(); + + if (StringUtils.equals( + op1Conf.getTableMetadata().getDbName() + "." + op1Conf.getTableMetadata().getTableName(), + op2Conf.getTableMetadata().getDbName() + "." + op2Conf.getTableMetadata().getTableName()) && + op1Conf.getNeededColumns().equals(op2Conf.getNeededColumns()) && + StringUtils.equals(op1Conf.getFilterExprString(), op2Conf.getFilterExprString()) && + pctx.getPrunedPartitions(tsOp1).getPartitions().equals( + pctx.getPrunedPartitions(tsOp2).getPartitions()) && + op1Conf.getRowLimit() == op2Conf.getRowLimit()) { + return true; + } else { + return false; + } + } + + OperatorComparatorFactory.OperatorComparator operatorComparator = + OperatorComparatorFactory.getOperatorComparator(op1.getClass()); + return operatorComparator.equals(op1, op2); + } + + private static boolean validPreConditions(ParseContext pctx, SharedWorkOptimizerCache optimizerCache, + SharedResult sr) { + + // We check whether merging the works would cause the size of + // the data in memory grow too large. + // TODO: Currently ignores GBY and PTF which may also buffer data in memory. + if (sr.dataSize > sr.maxDataSize) { + // Size surpasses limit, we cannot convert + LOG.debug("accumulated data size: {} / max size: {}", sr.dataSize, sr.maxDataSize); + return false; + } + + TableScanOperator tsOp1 = (TableScanOperator) sr.retainableOps.get(0); + TableScanOperator tsOp2 = (TableScanOperator) sr.discardableOps.get(0); + + // 1) The set of operators in the works of the TS operators need to meet + // some requirements. In particular: + // 1.1. None of the works that contain the TS operators can contain a Union + // operator. This is not supported yet as we might end up with cycles in + // the Tez DAG. + // 1.2. There cannot be more than one DummyStore operator in the new resulting + // work when the TS operators are merged. This is due to an assumption in + // MergeJoinProc that needs to be further explored. + // If any of these conditions are not met, we cannot merge. + // TODO: Extend rule so it can be applied for these cases. + final Set> workOps1 = findWorkOperators(optimizerCache, tsOp1); + final Set> workOps2 = findWorkOperators(optimizerCache, tsOp2); + boolean foundDummyStoreOp = false; + for (Operator op : workOps1) { + if (op instanceof UnionOperator) { + // We cannot merge (1.1) + return false; + } + if (op instanceof DummyStoreOperator) { + foundDummyStoreOp = true; + } + } + for (Operator op : workOps2) { + if (op instanceof UnionOperator) { + // We cannot merge (1.1) + return false; + } + if (foundDummyStoreOp && op instanceof DummyStoreOperator) { + // We cannot merge (1.2) + return false; + } + } + // 2) We check whether output works when we merge the operators will collide. + // + // Work1 Work2 (merge TS in W1 & W2) Work1 + // \ / -> | | X + // Work3 Work3 + // + // If we do, we cannot merge. The reason is that Tez currently does + // not support parallel edges, i.e., multiple edges from same work x + // into same work y. + final Set> outputWorksOps1 = findChildWorkOperators(pctx, optimizerCache, tsOp1); + final Set> outputWorksOps2 = findChildWorkOperators(pctx, optimizerCache, tsOp2); + if (!Collections.disjoint(outputWorksOps1, outputWorksOps2)) { + // We cannot merge + return false; + } + // 3) We check whether we will end up with same operators inputing on same work. + // + // Work1 (merge TS in W2 & W3) Work1 + // / \ -> | | X + // Work2 Work3 Work2 + // + // If we do, we cannot merge. The reason is the same as above, currently + // Tez currently does not support parallel edges. + final Set> inputWorksOps1 = findParentWorkOperators(pctx, optimizerCache, tsOp1); + final Set> inputWorksOps2 = + findParentWorkOperators(pctx, optimizerCache, tsOp2, sr.discardableInputOps); + if (!Collections.disjoint(inputWorksOps1, inputWorksOps2)) { + // We cannot merge + return false; + } + // 4) We check whether one of the operators is part of a work that is an input for + // the work of the other operator. + // + // Work1 (merge TS in W1 & W3) Work1 + // | -> | X + // Work2 Work2 + // | | + // Work3 Work1 + // + // If we do, we cannot merge, as we would end up with a cycle in the DAG. + final Set> descendantWorksOps1 = + findDescendantWorkOperators(pctx, optimizerCache, tsOp1, sr.discardableInputOps); + final Set> descendantWorksOps2 = + findDescendantWorkOperators(pctx, optimizerCache, tsOp2, sr.discardableInputOps); + if (!Collections.disjoint(descendantWorksOps1, workOps2) + || !Collections.disjoint(workOps1, descendantWorksOps2)) { + return false; + } + return true; + } + + private static Set> findParentWorkOperators(ParseContext pctx, + SharedWorkOptimizerCache optimizerCache, Operator start) { + return findParentWorkOperators(pctx, optimizerCache, start, ImmutableSet.of()); + } + + private static Set> findParentWorkOperators(ParseContext pctx, + SharedWorkOptimizerCache optimizerCache, Operator start, + Set> excludeOps) { + // Find operators in work + Set> workOps = findWorkOperators(optimizerCache, start); + // Gather input works operators + Set> set = new HashSet>(); + for (Operator op : workOps) { + if (op.getParentOperators() != null) { + for (Operator parent : op.getParentOperators()) { + if (parent instanceof ReduceSinkOperator && !excludeOps.contains(parent)) { + set.addAll(findWorkOperators(optimizerCache, parent)); + } + } + } else if (op instanceof TableScanOperator) { + // Check for DPP and semijoin DPP + for (Operator parent : optimizerCache.tableScanToDPPSource.get((TableScanOperator) op)) { + if (!excludeOps.contains(parent)) { + set.addAll(findWorkOperators(optimizerCache, parent)); + } + } + } + } + return set; + } + + private static Set> findAscendantWorkOperators(ParseContext pctx, + SharedWorkOptimizerCache optimizerCache, Operator start) { + // Find operators in work + Set> workOps = findWorkOperators(optimizerCache, start); + // Gather input works operators + Set> result = new HashSet>(); + Set> set; + while (!workOps.isEmpty()) { + set = new HashSet>(); + for (Operator op : workOps) { + if (op.getParentOperators() != null) { + for (Operator parent : op.getParentOperators()) { + if (parent instanceof ReduceSinkOperator) { + set.addAll(findWorkOperators(optimizerCache, parent)); + } + } + } else if (op instanceof TableScanOperator) { + // Check for DPP and semijoin DPP + for (Operator parent : optimizerCache.tableScanToDPPSource.get((TableScanOperator) op)) { + set.addAll(findWorkOperators(optimizerCache, parent)); + } + } + } + workOps = set; + result.addAll(set); + } + return result; + } + + private static Set> findChildWorkOperators(ParseContext pctx, + SharedWorkOptimizerCache optimizerCache, Operator start) { + // Find operators in work + Set> workOps = findWorkOperators(optimizerCache, start); + // Gather output works operators + Set> set = new HashSet>(); + for (Operator op : workOps) { + if (op instanceof ReduceSinkOperator) { + if (op.getChildOperators() != null) { + // All children of RS are descendants + for (Operator child : op.getChildOperators()) { + set.addAll(findWorkOperators(optimizerCache, child)); + } + } + // Semijoin DPP work is considered a child because work needs + // to finish for it to execute + SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op); + if (sjbi != null) { + set.addAll(findWorkOperators(optimizerCache, sjbi.getTsOp())); + } + } else if(op.getConf() instanceof DynamicPruningEventDesc) { + // DPP work is considered a child because work needs + // to finish for it to execute + set.addAll(findWorkOperators( + optimizerCache, ((DynamicPruningEventDesc) op.getConf()).getTableScan())); + } + } + return set; + } + + private static Set> findDescendantWorkOperators(ParseContext pctx, + SharedWorkOptimizerCache optimizerCache, Operator start, + Set> excludeOps) { + // Find operators in work + Set> workOps = findWorkOperators(optimizerCache, start); + // Gather output works operators + Set> result = new HashSet>(); + Set> set; + while (!workOps.isEmpty()) { + set = new HashSet>(); + for (Operator op : workOps) { + if (excludeOps.contains(op)) { + continue; + } + if (op instanceof ReduceSinkOperator) { + if (op.getChildOperators() != null) { + // All children of RS are descendants + for (Operator child : op.getChildOperators()) { + set.addAll(findWorkOperators(optimizerCache, child)); + } + } + // Semijoin DPP work is considered a descendant because work needs + // to finish for it to execute + SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op); + if (sjbi != null) { + set.addAll(findWorkOperators(optimizerCache, sjbi.getTsOp())); + } + } else if(op.getConf() instanceof DynamicPruningEventDesc) { + // DPP work is considered a descendant because work needs + // to finish for it to execute + set.addAll(findWorkOperators( + optimizerCache, ((DynamicPruningEventDesc) op.getConf()).getTableScan())); + } + } + workOps = set; + result.addAll(set); + } + return result; + } + + // Stores result in cache + private static Set> findWorkOperators( + SharedWorkOptimizerCache optimizerCache, Operator start) { + Set> c = optimizerCache.operatorToWorkOperators.get(start); + if (!c.isEmpty()) { + return c; + } + c = findWorkOperators(start, new HashSet>()); + for (Operator op : c) { + optimizerCache.operatorToWorkOperators.putAll(op, c); + } + return c; + } + + private static Set> findWorkOperators(Operator start, Set> found) { + found.add(start); + if (start.getParentOperators() != null) { + for (Operator parent : start.getParentOperators()) { + if (parent instanceof ReduceSinkOperator) { + continue; + } + if (!found.contains(parent)) { + findWorkOperators(parent, found); + } + } + } + if (start instanceof ReduceSinkOperator) { + return found; + } + if (start.getChildOperators() != null) { + for (Operator child : start.getChildOperators()) { + if (!found.contains(child)) { + findWorkOperators(child, found); + } + } + } + return found; + } + + private static void pushFilterToTopOfTableScan( + SharedWorkOptimizerCache optimizerCache, TableScanOperator tsOp) + throws UDFArgumentException { + ExprNodeGenericFuncDesc tableScanExprNode = tsOp.getConf().getFilterExpr(); + List> allChildren = + Lists.newArrayList(tsOp.getChildOperators()); + for (Operator op : allChildren) { + if (op instanceof FilterOperator) { + FilterOperator filterOp = (FilterOperator) op; + ExprNodeDesc filterExprNode = filterOp.getConf().getPredicate(); + if (tableScanExprNode.isSame(filterExprNode)) { + // We do not need to do anything + return; + } + if (tableScanExprNode.getGenericUDF() instanceof GenericUDFOPOr) { + for (ExprNodeDesc childExprNode : tableScanExprNode.getChildren()) { + if (childExprNode.isSame(filterExprNode)) { + // We do not need to do anything, it is in the OR expression + // so probably we pushed previously + return; + } + } + } + ExprNodeGenericFuncDesc newPred = ExprNodeGenericFuncDesc.newInstance( + new GenericUDFOPAnd(), + Arrays.asList(tableScanExprNode.clone(), filterExprNode)); + filterOp.getConf().setPredicate(newPred); + } else { + Operator newOp = OperatorFactory.get(tsOp.getCompilationOpContext(), + new FilterDesc(tableScanExprNode.clone(), false), + new RowSchema(tsOp.getSchema().getSignature())); + tsOp.replaceChild(op, newOp); + newOp.getParentOperators().add(tsOp); + op.replaceParent(tsOp, newOp); + newOp.getChildOperators().add(op); + // Add to cache (same group as tsOp) + optimizerCache.putIfWorkExists(newOp, tsOp); + } + } + } + + private static class SharedResult { + final List> retainableOps; + final List> discardableOps; + final Set> discardableInputOps; + final long dataSize; + final long maxDataSize; + + private SharedResult(Collection> retainableOps, Collection> discardableOps, + Set> discardableInputOps, long dataSize, long maxDataSize) { + this.retainableOps = ImmutableList.copyOf(retainableOps); + this.discardableOps = ImmutableList.copyOf(discardableOps); + this.discardableInputOps = ImmutableSet.copyOf(discardableInputOps); + this.dataSize = dataSize; + this.maxDataSize = maxDataSize; + } + } + + /** Cache to accelerate optimization */ + private static class SharedWorkOptimizerCache { + // Operators that belong to each work + final HashMultimap, Operator> operatorToWorkOperators = + HashMultimap., Operator>create(); + // Table scan operators to DPP sources + final Multimap> tableScanToDPPSource = + HashMultimap.>create(); + + // Add new operator to cache work group of existing operator (if group exists) + void putIfWorkExists(Operator opToAdd, Operator existingOp) { + List> c = ImmutableList.copyOf(operatorToWorkOperators.get(existingOp)); + if (!c.isEmpty()) { + for (Operator op : c) { + operatorToWorkOperators.get(op).add(opToAdd); + } + operatorToWorkOperators.putAll(opToAdd, c); + operatorToWorkOperators.put(opToAdd, opToAdd); + } + } + + // Remove operator + void removeOp(Operator opToRemove) { + Set> s = operatorToWorkOperators.get(opToRemove); + s.remove(opToRemove); + List> c1 = ImmutableList.copyOf(s); + if (!c1.isEmpty()) { + for (Operator op1 : c1) { + operatorToWorkOperators.remove(op1, opToRemove); // Remove operator + } + operatorToWorkOperators.removeAll(opToRemove); // Remove entry for operator + } + } + + // Remove operator and combine + void removeOpAndCombineWork(Operator opToRemove, Operator replacementOp) { + Set> s = operatorToWorkOperators.get(opToRemove); + s.remove(opToRemove); + List> c1 = ImmutableList.copyOf(s); + List> c2 = ImmutableList.copyOf(operatorToWorkOperators.get(replacementOp)); + if (!c1.isEmpty() && !c2.isEmpty()) { + for (Operator op1 : c1) { + operatorToWorkOperators.remove(op1, opToRemove); // Remove operator + operatorToWorkOperators.putAll(op1, c2); // Add ops of new collection + } + operatorToWorkOperators.removeAll(opToRemove); // Remove entry for operator + for (Operator op2 : c2) { + operatorToWorkOperators.putAll(op2, c1); // Add ops to existing collection + } + } + } + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index ca544b4..662bd40 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -275,7 +275,7 @@ public static void removeUnionOperators(GenTezProcContext context, BaseWork work SemiJoinBranchInfo sjInfo = rsToSemiJoinBranchInfo.get(rs); if (sjInfo.getTsOp() == orig) { SemiJoinBranchInfo newSJInfo = new SemiJoinBranchInfo( - (TableScanOperator)newRoot, sjInfo.getIsHint()); + (TableScanOperator) newRoot, sjInfo.getIsHint()); rsToSemiJoinBranchInfo.put(rs, newSJInfo); } } @@ -580,46 +580,149 @@ public static void removeSemiJoinOperator(ParseContext context, LOG.debug("Removing ReduceSink " + rs + " and TableScan " + ts); ExprNodeDesc constNode = new ExprNodeConstantDesc( TypeInfoFactory.booleanTypeInfo, Boolean.TRUE); + // TS operator DynamicValuePredicateContext filterDynamicValuePredicatesCollection = new DynamicValuePredicateContext(); - FilterDesc filterDesc = ((FilterOperator)(ts.getChildOperators().get(0))).getConf(); - collectDynamicValuePredicates(filterDesc.getPredicate(), - filterDynamicValuePredicatesCollection); - for (ExprNodeDesc nodeToRemove : filterDynamicValuePredicatesCollection - .childParentMapping.keySet()) { - // Find out if this synthetic predicate belongs to the current cycle - boolean skip = true; - for (ExprNodeDesc expr : nodeToRemove.getChildren()) { - if (expr instanceof ExprNodeDynamicValueDesc ) { - String dynamicValueIdFromExpr = ((ExprNodeDynamicValueDesc) expr) - .getDynamicValue().getId(); - List dynamicValueIdsFromMap = context. - getRsToRuntimeValuesInfoMap().get(rs).getDynamicValueIDs(); - for (String dynamicValueIdFromMap : dynamicValueIdsFromMap) { - if (dynamicValueIdFromExpr.equals(dynamicValueIdFromMap)) { - // Intended predicate to be removed - skip = false; - break; + if (ts.getConf().getFilterExpr() != null) { + collectDynamicValuePredicates(ts.getConf().getFilterExpr(), + filterDynamicValuePredicatesCollection); + for (ExprNodeDesc nodeToRemove : filterDynamicValuePredicatesCollection + .childParentMapping.keySet()) { + // Find out if this synthetic predicate belongs to the current cycle + boolean skip = true; + for (ExprNodeDesc expr : nodeToRemove.getChildren()) { + if (expr instanceof ExprNodeDynamicValueDesc ) { + String dynamicValueIdFromExpr = ((ExprNodeDynamicValueDesc) expr) + .getDynamicValue().getId(); + List dynamicValueIdsFromMap = context. + getRsToRuntimeValuesInfoMap().get(rs).getDynamicValueIDs(); + for (String dynamicValueIdFromMap : dynamicValueIdsFromMap) { + if (dynamicValueIdFromExpr.equals(dynamicValueIdFromMap)) { + // Intended predicate to be removed + skip = false; + break; + } } } } + if (!skip) { + ExprNodeDesc nodeParent = filterDynamicValuePredicatesCollection + .childParentMapping.get(nodeToRemove); + if (nodeParent == null) { + // This was the only predicate, set filter expression to null + ts.getConf().setFilterExpr(null); + } else { + int i = nodeParent.getChildren().indexOf(nodeToRemove); + nodeParent.getChildren().remove(i); + nodeParent.getChildren().add(i, constNode); + } + // skip the rest of the predicates + skip = true; + } + } + } + // Filter operator + filterDynamicValuePredicatesCollection = new DynamicValuePredicateContext(); + for (Operator op : ts.getChildOperators()) { + if (!(op instanceof FilterOperator)) { + continue; + } + FilterDesc filterDesc = ((FilterOperator) op).getConf(); + collectDynamicValuePredicates(filterDesc.getPredicate(), + filterDynamicValuePredicatesCollection); + for (ExprNodeDesc nodeToRemove : filterDynamicValuePredicatesCollection + .childParentMapping.keySet()) { + // Find out if this synthetic predicate belongs to the current cycle + boolean skip = true; + for (ExprNodeDesc expr : nodeToRemove.getChildren()) { + if (expr instanceof ExprNodeDynamicValueDesc ) { + String dynamicValueIdFromExpr = ((ExprNodeDynamicValueDesc) expr) + .getDynamicValue().getId(); + List dynamicValueIdsFromMap = context. + getRsToRuntimeValuesInfoMap().get(rs).getDynamicValueIDs(); + for (String dynamicValueIdFromMap : dynamicValueIdsFromMap) { + if (dynamicValueIdFromExpr.equals(dynamicValueIdFromMap)) { + // Intended predicate to be removed + skip = false; + break; + } + } + } + } + if (!skip) { + ExprNodeDesc nodeParent = filterDynamicValuePredicatesCollection + .childParentMapping.get(nodeToRemove); + if (nodeParent == null) { + // This was the only predicate, set filter expression to const + filterDesc.setPredicate(constNode); + } else { + int i = nodeParent.getChildren().indexOf(nodeToRemove); + nodeParent.getChildren().remove(i); + nodeParent.getChildren().add(i, constNode); + } + // skip the rest of the predicates + skip = true; + } + } + } + context.getRsToSemiJoinBranchInfo().remove(rs); + } + + // Functionality to remove semi-join optimization + public static void removeSemiJoinOperator(ParseContext context, + AppMasterEventOperator eventOp, TableScanOperator ts) throws SemanticException{ + // Cleanup the synthetic predicate in the tablescan operator and filter by + // replacing it with "true" + LOG.debug("Removing AppMasterEventOperator " + eventOp + " and TableScan " + ts); + ExprNodeDesc constNode = new ExprNodeConstantDesc( + TypeInfoFactory.booleanTypeInfo, Boolean.TRUE); + // Retrieve generator + DynamicPruningEventDesc dped = (DynamicPruningEventDesc) eventOp.getConf(); + // TS operator + DynamicPartitionPrunerContext filterDynamicListPredicatesCollection = + new DynamicPartitionPrunerContext(); + if (ts.getConf().getFilterExpr() != null) { + collectDynamicPruningConditions( + ts.getConf().getFilterExpr(), filterDynamicListPredicatesCollection); + for (DynamicListContext ctx : filterDynamicListPredicatesCollection) { + if (ctx.generator != dped.getGenerator()) { + continue; + } + // remove the condition by replacing it with "true" + if (ctx.grandParent == null) { + // This was the only predicate, set filter expression to const + ts.getConf().setFilterExpr(null); + } else { + int i = ctx.grandParent.getChildren().indexOf(ctx.parent); + ctx.grandParent.getChildren().remove(i); + ctx.grandParent.getChildren().add(i, constNode); + } } - if (!skip) { - ExprNodeDesc nodeParent = filterDynamicValuePredicatesCollection - .childParentMapping.get(nodeToRemove); - if (nodeParent == null) { + } + // Filter operator + filterDynamicListPredicatesCollection.dynLists.clear(); + for (Operator op : ts.getChildOperators()) { + if (!(op instanceof FilterOperator)) { + continue; + } + FilterDesc filterDesc = ((FilterOperator) op).getConf(); + collectDynamicPruningConditions( + filterDesc.getPredicate(), filterDynamicListPredicatesCollection); + for (DynamicListContext ctx : filterDynamicListPredicatesCollection) { + if (ctx.generator != dped.getGenerator()) { + continue; + } + // remove the condition by replacing it with "true" + if (ctx.grandParent == null) { // This was the only predicate, set filter expression to const filterDesc.setPredicate(constNode); } else { - int i = nodeParent.getChildren().indexOf(nodeToRemove); - nodeParent.getChildren().remove(i); - nodeParent.getChildren().add(i, constNode); + int i = ctx.grandParent.getChildren().indexOf(ctx.parent); + ctx.grandParent.getChildren().remove(i); + ctx.grandParent.getChildren().add(i, constNode); } - // skip the rest of the predicates - skip = true; } } - context.getRsToSemiJoinBranchInfo().remove(rs); } private static class DynamicValuePredicateContext implements NodeProcessorCtx { @@ -647,7 +750,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } } - private static void collectDynamicValuePredicates(ExprNodeDesc pred, NodeProcessorCtx ctx) throws SemanticException { + private static void collectDynamicValuePredicates(ExprNodeDesc pred, NodeProcessorCtx ctx) + throws SemanticException { // create a walker which walks the tree in a DFS manner while maintaining // the operator stack. The dispatcher // generates the plan from the operator tree @@ -660,4 +764,80 @@ private static void collectDynamicValuePredicates(ExprNodeDesc pred, NodeProcess egw.startWalking(startNodes, null); } + + public static class DynamicListContext { + public ExprNodeDynamicListDesc desc; + public ExprNodeDesc parent; + public ExprNodeDesc grandParent; + public ReduceSinkOperator generator; + + public DynamicListContext(ExprNodeDynamicListDesc desc, ExprNodeDesc parent, + ExprNodeDesc grandParent, ReduceSinkOperator generator) { + this.desc = desc; + this.parent = parent; + this.grandParent = grandParent; + this.generator = generator; + } + } + + public static class DynamicPartitionPrunerContext implements NodeProcessorCtx, + Iterable { + public List dynLists = new ArrayList(); + + public void addDynamicList(ExprNodeDynamicListDesc desc, ExprNodeDesc parent, + ExprNodeDesc grandParent, ReduceSinkOperator generator) { + dynLists.add(new DynamicListContext(desc, parent, grandParent, generator)); + } + + @Override + public Iterator iterator() { + return dynLists.iterator(); + } + } + + public static class DynamicPartitionPrunerProc implements NodeProcessor { + + /** + * process simply remembers all the dynamic partition pruning expressions + * found + */ + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + ExprNodeDynamicListDesc desc = (ExprNodeDynamicListDesc) nd; + DynamicPartitionPrunerContext context = (DynamicPartitionPrunerContext) procCtx; + + // Rule is searching for dynamic pruning expr. There's at least an IN + // expression wrapping it. + ExprNodeDesc parent = (ExprNodeDesc) stack.get(stack.size() - 2); + ExprNodeDesc grandParent = stack.size() >= 3 ? (ExprNodeDesc) stack.get(stack.size() - 3) : null; + + context.addDynamicList(desc, parent, grandParent, (ReduceSinkOperator) desc.getSource()); + + return context; + } + } + + public static Map collectDynamicPruningConditions(ExprNodeDesc pred, NodeProcessorCtx ctx) + throws SemanticException { + + // create a walker which walks the tree in a DFS manner while maintaining + // the operator stack. The dispatcher + // generates the plan from the operator tree + Map exprRules = new LinkedHashMap(); + exprRules.put(new RuleRegExp("R1", ExprNodeDynamicListDesc.class.getName() + "%"), + new DynamicPartitionPrunerProc()); + + // The dispatcher fires the processor corresponding to the closest matching + // rule and passes the context along + Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, ctx); + GraphWalker egw = new DefaultGraphWalker(disp); + + List startNodes = new ArrayList(); + startNodes.add(pred); + + HashMap outputMap = new HashMap(); + egw.startWalking(startNodes, outputMap); + return outputMap; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 20f16fb..888627a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -76,7 +76,7 @@ import org.apache.hadoop.hive.ql.optimizer.ReduceSinkMapJoinProc; import org.apache.hadoop.hive.ql.optimizer.RemoveDynamicPruningBySize; import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism; -import org.apache.hadoop.hive.ql.optimizer.SharedScanOptimizer; +import org.apache.hadoop.hive.ql.optimizer.SharedWorkOptimizer; import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits; import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer; import org.apache.hadoop.hive.ql.optimizer.physical.CrossProductCheck; @@ -184,8 +184,8 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run cycle analysis for partition pruning"); perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); - if(procCtx.conf.getBoolVar(ConfVars.HIVE_SHARED_SCAN_OPTIMIZATION)) { - new SharedScanOptimizer().transform(procCtx.parseContext); + if(procCtx.conf.getBoolVar(ConfVars.HIVE_SHARED_WORK_OPTIMIZATION)) { + new SharedWorkOptimizer().transform(procCtx.parseContext); } perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Shared scans optimization"); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java index 73bbebd..d88e110 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java @@ -20,6 +20,7 @@ import java.io.IOException; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -38,6 +39,9 @@ // tableScan is only available during compile private transient TableScanOperator tableScan; + // reduceSink is only available during compile + private transient ReduceSinkOperator generator; + // the partition column we're interested in private ExprNodeDesc partKey; @@ -49,6 +53,14 @@ public void setTableScan(TableScanOperator tableScan) { this.tableScan = tableScan; } + public ReduceSinkOperator getGenerator() { + return generator; + } + + public void setGenerator(ReduceSinkOperator generator) { + this.generator = generator; + } + @Explain(displayName = "Target column") public String displayTargetColumn() { return targetColumnName + " (" + targetColumnType + ")"; diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java index 04686f7..570bd6b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java @@ -176,6 +176,9 @@ public boolean isAcidTable() { @Explain(displayName = "filterExpr") public String getFilterExprString() { + if (filterExpr == null) { + return null; + } return PlanUtils.getExprListString(Arrays.asList(filterExpr)); } diff --git ql/src/test/results/clientpositive/llap/auto_join0.q.out ql/src/test/results/clientpositive/llap/auto_join0.q.out index 6d051ea..1141db2 100644 --- ql/src/test/results/clientpositive/llap/auto_join0.q.out +++ ql/src/test/results/clientpositive/llap/auto_join0.q.out @@ -52,13 +52,6 @@ STAGE PLANS: key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key < 10) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ diff --git ql/src/test/results/clientpositive/llap/auto_join30.q.out ql/src/test/results/clientpositive/llap/auto_join30.q.out index 90c4241..b1cb5fa 100644 --- ql/src/test/results/clientpositive/llap/auto_join30.q.out +++ ql/src/test/results/clientpositive/llap/auto_join30.q.out @@ -497,13 +497,6 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + @@ -678,10 +671,6 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + @@ -862,10 +851,6 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + @@ -1230,10 +1215,6 @@ STAGE PLANS: sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out index bdb30d7..b53b2ed 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out @@ -474,10 +474,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Map 1 <- Map 4 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (BROADCAST_EDGE) + Reducer 3 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -500,7 +499,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Map 4 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -514,9 +513,15 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Map 4 Map Operator Tree: TableScan alias: b @@ -533,54 +538,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0 - input vertices: - 1 Map 3 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -600,7 +557,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Reducer 5 + 1 Reducer 3 Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) @@ -613,7 +570,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Reducer 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2298,10 +2255,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Map 4 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE) + Map 1 <- Map 4 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (BROADCAST_EDGE) + Reducer 3 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2324,7 +2280,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0 input vertices: - 1 Map 3 + 1 Map 4 Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() @@ -2338,9 +2294,15 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs - Map 3 + Map 4 Map Operator Tree: TableScan alias: b @@ -2357,54 +2319,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0 - input vertices: - 1 Map 3 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -2424,7 +2338,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 input vertices: - 1 Reducer 5 + 1 Reducer 3 Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint) @@ -2437,7 +2351,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 + Reducer 3 Execution mode: llap Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out index 042c60b..649afb5 100644 --- ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out @@ -818,13 +818,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out index cdae4fb..8e85c02 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out @@ -1780,9 +1780,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(value) keys: key (type: string) @@ -1964,9 +1961,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(value) keys: key (type: string) diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out index 3e71546..21de860 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out @@ -45,13 +45,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -273,13 +266,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -716,13 +702,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -944,13 +923,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out index 82dae9a..c09a076 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out @@ -1815,13 +1815,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -2040,13 +2033,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -2265,13 +2251,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -2490,13 +2469,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index 2875e13..949d828 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -2239,27 +2239,21 @@ STAGE PLANS: alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '2008-04-08') (type: boolean) - Select Operator + Select Operator + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - sort order: - Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '2008-04-08') (type: boolean) - Select Operator - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: '2008-04-08' (type: string) - mode: hash - outputColumnNames: _col0 + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 diff --git ql/src/test/results/clientpositive/llap/except_distinct.q.out ql/src/test/results/clientpositive/llap/except_distinct.q.out index e4c2941..4dd596e 100644 --- ql/src/test/results/clientpositive/llap/except_distinct.q.out +++ ql/src/test/results/clientpositive/llap/except_distinct.q.out @@ -398,10 +398,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) @@ -414,10 +410,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(1) keys: _col0 (type: string), _col1 (type: string) diff --git ql/src/test/results/clientpositive/llap/explainuser_1.q.out ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 28ca13b..e9a620d 100644 --- ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -2448,11 +2448,7 @@ Stage-0 Output:["_col0"],aggregations:["avg(VALUE._col0)"] <-Map 5 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_18] - Group By Operator [GBY_17] (rows=1 width=76) - Output:["_col0"],aggregations:["avg(p_size)"] - Filter Operator [FIL_35] (rows=8 width=4) - predicate:(p_size < 10) - Please refer to the previous TableScan [TS_2] + Please refer to the previous Group By Operator [GBY_5] PREHOOK: query: explain select b.p_mfgr, min(p_retailprice) from part b @@ -2541,9 +2537,7 @@ Stage-0 PartitionCols:_col0 Group By Operator [GBY_8] (rows=5 width=114) Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr - Select Operator [SEL_7] (rows=26 width=106) - Output:["p_mfgr","p_retailprice"] - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_1] <-Reducer 8 [SIMPLE_EDGE] llap SHUFFLE [RS_31] PartitionCols:_col0, _col1 diff --git ql/src/test/results/clientpositive/llap/explainuser_2.q.out ql/src/test/results/clientpositive/llap/explainuser_2.q.out index a250fd6..6894121 100644 --- ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -490,9 +490,9 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] llap SHUFFLE [RS_46] PartitionCols:_col1 - Select Operator [SEL_41] (rows=25 width=175) + Select Operator [SEL_15] (rows=25 width=175) Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=25 width=175) + Filter Operator [FIL_76] (rows=25 width=175) predicate:(key is not null and value is not null) TableScan [TS_13] (rows=25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -552,11 +552,7 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] llap SHUFFLE [RS_20] PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_76] (rows=25 width=175) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_19] PartitionCols:_col1 @@ -662,9 +658,9 @@ Stage-0 <-Map 16 [SIMPLE_EDGE] llap SHUFFLE [RS_112] PartitionCols:_col0 - Select Operator [SEL_107] (rows=500 width=178) + Select Operator [SEL_18] (rows=500 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=500 width=178) + Filter Operator [FIL_153] (rows=500 width=178) predicate:key is not null TableScan [TS_16] (rows=500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -676,9 +672,9 @@ Stage-0 <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_109] PartitionCols:_col1 - Select Operator [SEL_104] (rows=25 width=175) + Select Operator [SEL_15] (rows=25 width=175) Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=25 width=175) + Filter Operator [FIL_152] (rows=25 width=175) predicate:(key is not null and value is not null) TableScan [TS_13] (rows=25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -788,11 +784,7 @@ Stage-0 <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_56] PartitionCols:_col1 - Select Operator [SEL_51] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_157] (rows=25 width=175) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Reducer 21 [SIMPLE_EDGE] llap SHUFFLE [RS_55] PartitionCols:_col1 @@ -856,11 +848,7 @@ Stage-0 <-Map 16 [SIMPLE_EDGE] llap SHUFFLE [RS_23] PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500 width=178) - predicate:key is not null - Please refer to the previous TableScan [TS_16] + Please refer to the previous Select Operator [SEL_18] <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_22] PartitionCols:_col2 @@ -869,11 +857,7 @@ Stage-0 <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_20] PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=25 width=175) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_19] PartitionCols:_col1 @@ -1346,9 +1330,9 @@ Stage-0 <-Map 10 [BROADCAST_EDGE] llap BROADCAST [RS_112] PartitionCols:_col0 - Select Operator [SEL_107] (rows=500 width=10) + Select Operator [SEL_18] (rows=500 width=10) Output:["_col0","_col1"] - Filter Operator [FIL_164] (rows=500 width=10) + Filter Operator [FIL_153] (rows=500 width=10) predicate:key is not null TableScan [TS_16] (rows=500 width=10) default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -1357,9 +1341,9 @@ Stage-0 <-Map 9 [BROADCAST_EDGE] llap BROADCAST [RS_109] PartitionCols:_col1 - Select Operator [SEL_104] (rows=25 width=7) + Select Operator [SEL_15] (rows=25 width=7) Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=25 width=7) + Filter Operator [FIL_152] (rows=25 width=7) predicate:(key is not null and value is not null) TableScan [TS_13] (rows=25 width=7) default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -1529,21 +1513,13 @@ Stage-0 <-Map 10 [BROADCAST_EDGE] llap BROADCAST [RS_23] PartitionCols:_col0 - Select Operator [SEL_18] (rows=500 width=10) - Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=500 width=10) - predicate:key is not null - Please refer to the previous TableScan [TS_16] + Please refer to the previous Select Operator [SEL_18] <-Map Join Operator [MAPJOIN_165] (rows=288 width=10) Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col2"] <-Map 9 [BROADCAST_EDGE] llap BROADCAST [RS_20] PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_152] (rows=25 width=7) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Select Operator [SEL_12] (rows=262 width=10) Output:["_col1"] Group By Operator [GBY_11] (rows=262 width=10) @@ -2067,8 +2043,8 @@ Stage-0 <-Map 6 [BROADCAST_EDGE] llap BROADCAST [RS_69] PartitionCols:_col1 - Map Join Operator [MAPJOIN_118] (rows=27 width=7) - Conds:SEL_48._col0=RS_67._col0(Inner),Output:["_col0","_col1","_col3"] + Map Join Operator [MAPJOIN_122] (rows=27 width=7) + Conds:SEL_10._col0=RS_71._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 15 [BROADCAST_EDGE] llap BROADCAST [RS_67] PartitionCols:_col0 @@ -2078,9 +2054,9 @@ Stage-0 predicate:key is not null TableScan [TS_49] (rows=25 width=7) default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_48] (rows=25 width=7) + <-Select Operator [SEL_10] (rows=25 width=7) Output:["_col0","_col1"] - Filter Operator [FIL_108] (rows=25 width=7) + Filter Operator [FIL_105] (rows=25 width=7) predicate:(key is not null and value is not null) TableScan [TS_8] (rows=25 width=7) default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -2162,11 +2138,7 @@ Stage-0 <-Map 6 [BROADCAST_EDGE] llap BROADCAST [RS_15] PartitionCols:_col1 - Select Operator [SEL_10] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_101] (rows=25 width=7) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_8] + Please refer to the previous Select Operator [SEL_10] <-Select Operator [SEL_2] (rows=25 width=7) Output:["_col0"] Filter Operator [FIL_99] (rows=25 width=7) @@ -2318,7 +2290,7 @@ Stage-0 BROADCAST [RS_111] PartitionCols:_col1 Map Join Operator [MAPJOIN_166] (rows=27 width=7) - Conds:SEL_71._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"] + Conds:SEL_15._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 22 [BROADCAST_EDGE] llap BROADCAST [RS_109] PartitionCols:_col0 @@ -2328,9 +2300,9 @@ Stage-0 predicate:key is not null TableScan [TS_72] (rows=25 width=7) default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_71] (rows=25 width=7) + <-Select Operator [SEL_15] (rows=25 width=7) Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=25 width=7) + Filter Operator [FIL_149] (rows=25 width=7) predicate:(key is not null and value is not null) TableScan [TS_13] (rows=25 width=7) default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -2519,11 +2491,7 @@ Stage-0 <-Map 10 [BROADCAST_EDGE] llap BROADCAST [RS_20] PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_149] (rows=25 width=7) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Select Operator [SEL_12] (rows=262 width=10) Output:["_col1"] Group By Operator [GBY_11] (rows=262 width=10) @@ -2650,8 +2618,8 @@ Stage-5 <-Map 6 [BROADCAST_EDGE] llap BROADCAST [RS_69] PartitionCols:_col1 - Map Join Operator [MAPJOIN_123] (rows=27 width=7) - Conds:SEL_48._col0=RS_67._col0(Inner),Output:["_col0","_col1","_col3"] + Map Join Operator [MAPJOIN_127] (rows=27 width=7) + Conds:SEL_10._col0=RS_71._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 15 [BROADCAST_EDGE] llap BROADCAST [RS_67] PartitionCols:_col0 @@ -2661,9 +2629,9 @@ Stage-5 predicate:key is not null TableScan [TS_49] (rows=25 width=7) default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_48] (rows=25 width=7) + <-Select Operator [SEL_10] (rows=25 width=7) Output:["_col0","_col1"] - Filter Operator [FIL_113] (rows=25 width=7) + Filter Operator [FIL_110] (rows=25 width=7) predicate:(key is not null and value is not null) TableScan [TS_8] (rows=25 width=7) default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -2773,11 +2741,7 @@ Stage-5 <-Map 6 [BROADCAST_EDGE] llap BROADCAST [RS_15] PartitionCols:_col1 - Select Operator [SEL_10] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_106] (rows=25 width=7) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_8] + Please refer to the previous Select Operator [SEL_10] <-Select Operator [SEL_2] (rows=25 width=7) Output:["_col0"] Filter Operator [FIL_104] (rows=25 width=7) @@ -2965,7 +2929,7 @@ Stage-5 BROADCAST [RS_111] PartitionCols:_col1 Map Join Operator [MAPJOIN_171] (rows=27 width=7) - Conds:SEL_71._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"] + Conds:SEL_15._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"] <-Map 22 [BROADCAST_EDGE] llap BROADCAST [RS_109] PartitionCols:_col0 @@ -2975,9 +2939,9 @@ Stage-5 predicate:key is not null TableScan [TS_72] (rows=25 width=7) default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"] - <-Select Operator [SEL_71] (rows=25 width=7) + <-Select Operator [SEL_15] (rows=25 width=7) Output:["_col0","_col1"] - Filter Operator [FIL_161] (rows=25 width=7) + Filter Operator [FIL_154] (rows=25 width=7) predicate:(key is not null and value is not null) TableScan [TS_13] (rows=25 width=7) default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"] @@ -3166,11 +3130,7 @@ Stage-5 <-Map 10 [BROADCAST_EDGE] llap BROADCAST [RS_20] PartitionCols:_col1 - Select Operator [SEL_15] (rows=25 width=7) - Output:["_col0","_col1"] - Filter Operator [FIL_154] (rows=25 width=7) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Select Operator [SEL_12] (rows=262 width=10) Output:["_col1"] Group By Operator [GBY_11] (rows=262 width=10) diff --git ql/src/test/results/clientpositive/llap/intersect_merge.q.out ql/src/test/results/clientpositive/llap/intersect_merge.q.out index a312966..cf86a32 100644 --- ql/src/test/results/clientpositive/llap/intersect_merge.q.out +++ ql/src/test/results/clientpositive/llap/intersect_merge.q.out @@ -1764,16 +1764,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Select Operator - expressions: key (type: int), value (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: int) sort order: ++ diff --git ql/src/test/results/clientpositive/llap/limit_pushdown.q.out ql/src/test/results/clientpositive/llap/limit_pushdown.q.out index 57594e0..d1d6664 100644 --- ql/src/test/results/clientpositive/llap/limit_pushdown.q.out +++ ql/src/test/results/clientpositive/llap/limit_pushdown.q.out @@ -932,16 +932,6 @@ STAGE PLANS: Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.3 value expressions: _col1 (type: bigint) - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + diff --git ql/src/test/results/clientpositive/llap/mrr.q.out ql/src/test/results/clientpositive/llap/mrr.q.out index 726349c..7866e37 100644 --- ql/src/test/results/clientpositive/llap/mrr.q.out +++ ql/src/test/results/clientpositive/llap/mrr.q.out @@ -1322,30 +1322,12 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(value) - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + diff --git ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out index b4b0e93..daf0189 100644 --- ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out +++ ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out @@ -722,7 +722,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 6 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE) + Map 1 <- Map 6 (BROADCAST_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) @@ -761,28 +761,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - input vertices: - 1 Map 7 - Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -809,25 +787,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -982,7 +941,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 6 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE) + Map 1 <- Map 6 (BROADCAST_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) @@ -1021,28 +980,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col0 - input vertices: - 1 Map 7 - Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -1069,25 +1006,6 @@ STAGE PLANS: Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 7 - Map Operator Tree: - TableScan - alias: y1 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -1300,9 +1218,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: key (type: string) mode: hash @@ -1527,9 +1442,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: key (type: string) mode: hash @@ -1933,13 +1845,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Inner Join 0 to 1 diff --git ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out index c89ca6b..4b3a846 100644 --- ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out +++ ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out @@ -1319,10 +1319,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 2.0E-5 - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + diff --git ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out index 34ba87c..bd3d247 100644 --- ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out +++ ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out @@ -203,13 +203,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 10 Data size: 170 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: i is not null (type: boolean) - Statistics: Num rows: 10 Data size: 170 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: i (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 170 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + diff --git ql/src/test/results/clientpositive/llap/subquery_in.q.out ql/src/test/results/clientpositive/llap/subquery_in.q.out index 1f9c9e4..aa8b1aa 100644 --- ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -4024,9 +4024,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: p_type (type: string) @@ -4494,9 +4491,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: p_type (type: string) @@ -4509,9 +4503,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: p_type (type: string) @@ -4775,15 +4766,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 1040 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: struct) - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: avg(p_size) - keys: p_partkey (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1040 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + diff --git ql/src/test/results/clientpositive/llap/subquery_multi.q.out ql/src/test/results/clientpositive/llap/subquery_multi.q.out index 29516ef..43f8a6f 100644 --- ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -1710,14 +1710,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_brand (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ @@ -1763,13 +1755,6 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string), p_container (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string), _col0 (type: string) sort order: ++ @@ -2248,14 +2233,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_type (type: string), p_brand (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ @@ -2301,13 +2278,6 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_brand (type: string), p_type (type: string), p_container (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string), _col0 (type: string) sort order: ++ @@ -2631,9 +2601,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 260 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Filter Operator - predicate: p_size is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_type (type: string), p_size (type: int) mode: hash @@ -3080,15 +3047,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Select Operator - expressions: l_quantity (type: double) - outputColumnNames: l_quantity - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: avg(l_quantity) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE diff --git ql/src/test/results/clientpositive/llap/subquery_notin.q.out ql/src/test/results/clientpositive/llap/subquery_notin.q.out index b4af915..2bd9c9f 100644 --- ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -933,18 +933,12 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Filter Operator - predicate: (p_mfgr = p_mfgr) (type: boolean) - Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: p_mfgr (type: string), p_size (type: int) sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Filter Operator - predicate: (p_mfgr = p_mfgr) (type: boolean) - Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: p_mfgr (type: string), p_size (type: int) sort order: ++ @@ -1507,9 +1501,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Filter Operator - predicate: (key < '11') (type: boolean) - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: CASE WHEN ((key > '104')) THEN (null) ELSE (key) END (type: string) outputColumnNames: _col0 @@ -1699,9 +1690,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: (p_size * p_size) (type: int), p_type (type: string) outputColumnNames: _col0, _col1 @@ -1896,16 +1884,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -2121,16 +2099,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Select Operator - expressions: p_partkey (type: int), p_type (type: string) - outputColumnNames: p_partkey, p_type - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_partkey) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -2356,9 +2324,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1781 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Filter Operator - predicate: p_name is not null (type: boolean) - Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_partkey (type: int), p_name (type: string) mode: hash @@ -2583,16 +2548,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: double) - Select Operator - expressions: p_type (type: string), p_retailprice (type: double) - outputColumnNames: p_type, p_retailprice - Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_retailprice) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -2790,10 +2745,6 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int), _col0 (type: int) Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Select Operator - expressions: p_partkey (type: int), p_name (type: string), p_size (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int), _col0 (type: int) sort order: ++ @@ -2817,11 +2768,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: p_size (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + @@ -3072,9 +3018,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 3 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Filter Operator - predicate: ((p_size = p_size) and (p_partkey = p_partkey)) (type: boolean) - Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_partkey (type: int), p_name (type: string), p_size (type: int) mode: hash @@ -3244,9 +3187,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Filter Operator - predicate: p_brand is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: (UDFToDouble(p_type) + 2.0) (type: double), p_brand (type: string) outputColumnNames: _col0, _col1 @@ -3411,10 +3351,6 @@ STAGE PLANS: Map-reduce partition columns: (_col1 + 1) (type: int) Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: (_col1 + 1) (type: int) sort order: + @@ -3442,15 +3378,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (p_size + 1) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + @@ -4279,16 +4206,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -4517,16 +4434,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Select Operator - expressions: p_type (type: string), p_size (type: int) - outputColumnNames: p_type, p_size - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -5462,9 +5369,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Filter Operator - predicate: p_brand is not null (type: boolean) - Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_brand (type: string), p_type (type: string) mode: hash @@ -5872,10 +5776,6 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int) - Select Operator - expressions: c1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: UDFToDouble(_col0) (type: double) sort order: + @@ -5899,11 +5799,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: char(100)) Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: c2 (type: char(100)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: char(100)) sort order: + @@ -6184,9 +6079,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Filter Operator - predicate: b is not null (type: boolean) - Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: b (type: int), a (type: int) mode: hash @@ -6403,9 +6295,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Filter Operator - predicate: j is not null (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: i (type: int), j (type: int) mode: hash @@ -7319,11 +7208,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: value (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + diff --git ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out index bff2781..dde2384 100644 --- ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out +++ ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out @@ -71,8 +71,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Filter Operator predicate: false (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE @@ -85,8 +83,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Select Operator - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Filter Operator predicate: false (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE diff --git ql/src/test/results/clientpositive/llap/subquery_scalar.q.out ql/src/test/results/clientpositive/llap/subquery_scalar.q.out index e94edff..c3a53eb 100644 --- ql/src/test/results/clientpositive/llap/subquery_scalar.q.out +++ ql/src/test/results/clientpositive/llap/subquery_scalar.q.out @@ -376,11 +376,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) - Filter Operator - predicate: p_name is null (type: boolean) - Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE @@ -830,10 +825,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Select Operator - expressions: p_size (type: int) - outputColumnNames: p_size - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: avg(p_size) mode: hash @@ -2498,11 +2489,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key = '90') (type: boolean) - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: '90' (type: string) @@ -2873,15 +2859,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Select Operator - expressions: p_brand (type: string) - outputColumnNames: p_brand - Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(p_brand) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -2906,15 +2883,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(p_name) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE diff --git ql/src/test/results/clientpositive/llap/subquery_select.q.out ql/src/test/results/clientpositive/llap/subquery_select.q.out index 202980e..ce7c7e0 100644 --- ql/src/test/results/clientpositive/llap/subquery_select.q.out +++ ql/src/test/results/clientpositive/llap/subquery_select.q.out @@ -2913,10 +2913,6 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_size (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + @@ -3118,15 +3114,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Filter Operator - predicate: p_type is not null (type: boolean) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(p_size) - keys: p_type (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -3153,15 +3140,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Select Operator - expressions: p_name (type: string) - outputColumnNames: p_name - Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(p_name) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE @@ -3590,8 +3568,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE @@ -4449,10 +4425,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Select Operator - expressions: p_size (type: int) - outputColumnNames: p_size - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(p_size) mode: hash @@ -4462,10 +4434,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Select Operator - expressions: p_size (type: int) - outputColumnNames: p_size - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: avg(p_size) mode: hash @@ -4475,10 +4443,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Select Operator - expressions: p_size (type: int) - outputColumnNames: p_size - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: sum(p_size) mode: hash @@ -5243,9 +5207,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: p_partkey (type: int) @@ -5258,9 +5219,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 156 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Filter Operator - predicate: p_partkey is not null (type: boolean) - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: p_partkey (type: int) mode: hash diff --git ql/src/test/results/clientpositive/llap/subquery_views.q.out ql/src/test/results/clientpositive/llap/subquery_views.q.out index 1a21a02..5905ef0 100644 --- ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -243,11 +243,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + diff --git ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out index b4b6019..ead0909 100644 --- ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out +++ ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out @@ -56,13 +56,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -289,13 +282,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -5395,11 +5381,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE @@ -11369,19 +11350,9 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE diff --git ql/src/test/results/clientpositive/llap/union_top_level.q.out ql/src/test/results/clientpositive/llap/union_top_level.q.out index 2fac8cc..268e041 100644 --- ql/src/test/results/clientpositive/llap/union_top_level.q.out +++ ql/src/test/results/clientpositive/llap/union_top_level.q.out @@ -232,13 +232,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -264,13 +257,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + diff --git ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out index e1ad06c..4f914c8 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out @@ -77,15 +77,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 8 Data size: 2040 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint) - Filter Operator - predicate: (UDFToDouble(a) < 3.0) (type: boolean) - Statistics: Num rows: 2 Data size: 510 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: a (type: string), b (type: string), 0 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 2040 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) sort order: +++ @@ -214,15 +205,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) Statistics: Num rows: 8 Data size: 2040 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint) - Filter Operator - predicate: (UDFToDouble(a) < 3.0) (type: boolean) - Statistics: Num rows: 2 Data size: 510 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: a (type: string), b (type: string), 0 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 2040 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) sort order: +++ @@ -383,15 +365,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 2 Data size: 510 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Filter Operator - predicate: (UDFToDouble(a) < 3.0) (type: boolean) - Statistics: Num rows: 2 Data size: 510 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - keys: a (type: string), b (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 510 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ diff --git ql/src/test/results/clientpositive/llap/vector_join30.q.out ql/src/test/results/clientpositive/llap/vector_join30.q.out index ec76750..6b544f3 100644 --- ql/src/test/results/clientpositive/llap/vector_join30.q.out +++ ql/src/test/results/clientpositive/llap/vector_join30.q.out @@ -854,21 +854,6 @@ STAGE PLANS: nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Filter Operator - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsNotNull(col 0) -> boolean - predicate: key is not null (type: boolean) - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumns: [0, 1] - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) sort order: + @@ -1162,14 +1147,6 @@ STAGE PLANS: nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumns: [0, 1] - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) sort order: + @@ -1461,14 +1438,6 @@ STAGE PLANS: nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumns: [0, 1] - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) sort order: + @@ -2059,14 +2028,6 @@ STAGE PLANS: nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumns: [0, 1] - Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: string) sort order: + diff --git ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index d9fc6b5..6a8dffe 100644 --- ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -2386,27 +2386,21 @@ STAGE PLANS: alias: srcpart filterExpr: (ds = '2008-04-08') (type: boolean) Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '2008-04-08') (type: boolean) - Select Operator + Select Operator + Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: '2008-04-08' (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - sort order: - Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (ds = '2008-04-08') (type: boolean) - Select Operator - Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: '2008-04-08' (type: string) - mode: hash - outputColumnNames: _col0 + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: no inputs Map Vectorization: diff --git ql/src/test/results/clientpositive/perf/query1.q.out ql/src/test/results/clientpositive/perf/query1.q.out index da4a65c..9c5d440 100644 --- ql/src/test/results/clientpositive/perf/query1.q.out +++ ql/src/test/results/clientpositive/perf/query1.q.out @@ -165,9 +165,5 @@ Stage-0 <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_27] PartitionCols:_col0 - Select Operator [SEL_25] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_73] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] diff --git ql/src/test/results/clientpositive/perf/query10.q.out ql/src/test/results/clientpositive/perf/query10.q.out index 9b6621c..73c474a 100644 --- ql/src/test/results/clientpositive/perf/query10.q.out +++ ql/src/test/results/clientpositive/perf/query10.q.out @@ -174,9 +174,9 @@ Stage-0 <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_55] PartitionCols:_col0 - Select Operator [SEL_53] (rows=4058 width=1119) + Select Operator [SEL_14] (rows=4058 width=1119) Output:["_col0"] - Filter Operator [FIL_105] (rows=4058 width=1119) + Filter Operator [FIL_101] (rows=4058 width=1119) predicate:((d_year = 2002) and d_moy BETWEEN 4 AND 7 and d_date_sk is not null) TableScan [TS_12] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] @@ -213,11 +213,7 @@ Stage-0 <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_14] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_101] (rows=4058 width=1119) - predicate:((d_year = 2002) and d_moy BETWEEN 4 AND 7 and d_date_sk is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col0 @@ -242,11 +238,7 @@ Stage-0 <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col0 - Select Operator [SEL_28] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_103] (rows=4058 width=1119) - predicate:((d_year = 2002) and d_moy BETWEEN 4 AND 7 and d_date_sk is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_29] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query11.q.out ql/src/test/results/clientpositive/perf/query11.q.out index cce6350..11a4a6b 100644 --- ql/src/test/results/clientpositive/perf/query11.q.out +++ ql/src/test/results/clientpositive/perf/query11.q.out @@ -205,9 +205,9 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_34] PartitionCols:_col0 - Select Operator [SEL_29] (rows=80000000 width=860) + Select Operator [SEL_8] (rows=80000000 width=860) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_158] (rows=80000000 width=860) + Filter Operator [FIL_155] (rows=80000000 width=860) predicate:(c_customer_sk is not null and c_customer_id is not null) TableScan [TS_6] (rows=80000000 width=860) default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_first_name","c_last_name","c_preferred_cust_flag","c_birth_country","c_login","c_email_address"] @@ -253,11 +253,7 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_78] PartitionCols:_col0 - Select Operator [SEL_73] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_164] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_customer_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 14 [SIMPLE_EDGE] SHUFFLE [RS_77] PartitionCols:_col1 @@ -274,11 +270,7 @@ Stage-0 <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_74] PartitionCols:_col0 - Select Operator [SEL_67] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_162] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - Please refer to the previous TableScan [TS_21] + Please refer to the previous Select Operator [SEL_23] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_86] PartitionCols:_col0 @@ -298,11 +290,7 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_155] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_customer_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col1 @@ -346,11 +334,7 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_56] PartitionCols:_col0 - Select Operator [SEL_51] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_161] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_customer_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 7 [SIMPLE_EDGE] SHUFFLE [RS_55] PartitionCols:_col1 @@ -367,9 +351,5 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_52] PartitionCols:_col0 - Select Operator [SEL_45] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_159] (rows=144002668 width=135) - predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] diff --git ql/src/test/results/clientpositive/perf/query14.q.out ql/src/test/results/clientpositive/perf/query14.q.out index 64c5921..eab635e 100644 --- ql/src/test/results/clientpositive/perf/query14.q.out +++ ql/src/test/results/clientpositive/perf/query14.q.out @@ -1,6 +1,6 @@ +Warning: Shuffle Join MERGEJOIN[899][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product Warning: Shuffle Join MERGEJOIN[900][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 12' is a cross product Warning: Shuffle Join MERGEJOIN[901][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 16' is a cross product -Warning: Shuffle Join MERGEJOIN[899][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product PREHOOK: query: explain with cross_items as (select i_item_sk ss_item_sk @@ -337,18 +337,18 @@ Stage-0 <-Map 76 [SIMPLE_EDGE] SHUFFLE [RS_208] PartitionCols:_col0 - Select Operator [SEL_206] (rows=8116 width=1119) + Select Operator [SEL_15] (rows=8116 width=1119) Output:["_col0"] - Filter Operator [FIL_807] (rows=8116 width=1119) + Filter Operator [FIL_782] (rows=8116 width=1119) predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) TableScan [TS_13] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] <-Map 72 [SIMPLE_EDGE] SHUFFLE [RS_207] PartitionCols:_col0 - Select Operator [SEL_203] (rows=287989836 width=135) + Select Operator [SEL_12] (rows=287989836 width=135) Output:["_col0","_col1"] - Filter Operator [FIL_806] (rows=287989836 width=135) + Filter Operator [FIL_781] (rows=287989836 width=135) predicate:cs_sold_date_sk is not null TableScan [TS_10] (rows=287989836 width=135) default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity"] @@ -365,18 +365,18 @@ Stage-0 <-Map 84 [SIMPLE_EDGE] SHUFFLE [RS_219] PartitionCols:_col0 - Select Operator [SEL_217] (rows=8116 width=1119) + Select Operator [SEL_26] (rows=8116 width=1119) Output:["_col0"] - Filter Operator [FIL_809] (rows=8116 width=1119) + Filter Operator [FIL_784] (rows=8116 width=1119) predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) TableScan [TS_24] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] <-Map 80 [SIMPLE_EDGE] SHUFFLE [RS_218] PartitionCols:_col0 - Select Operator [SEL_214] (rows=144002668 width=135) + Select Operator [SEL_23] (rows=144002668 width=135) Output:["_col0","_col1"] - Filter Operator [FIL_808] (rows=144002668 width=135) + Filter Operator [FIL_783] (rows=144002668 width=135) predicate:ws_sold_date_sk is not null TableScan [TS_21] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity"] @@ -393,18 +393,18 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_198] PartitionCols:_col0 - Select Operator [SEL_196] (rows=8116 width=1119) + Select Operator [SEL_5] (rows=8116 width=1119) Output:["_col0"] - Filter Operator [FIL_805] (rows=8116 width=1119) + Filter Operator [FIL_780] (rows=8116 width=1119) predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_197] PartitionCols:_col0 - Select Operator [SEL_193] (rows=575995635 width=88) + Select Operator [SEL_2] (rows=575995635 width=88) Output:["_col0","_col1"] - Filter Operator [FIL_804] (rows=575995635 width=88) + Filter Operator [FIL_779] (rows=575995635 width=88) predicate:ss_sold_date_sk is not null TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity"] @@ -426,17 +426,13 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_242] PartitionCols:_col0 - Select Operator [SEL_240] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_811] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 88 [SIMPLE_EDGE] SHUFFLE [RS_241] PartitionCols:_col0 - Select Operator [SEL_237] (rows=575995635 width=88) + Select Operator [SEL_46] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_810] (rows=575995635 width=88) + Filter Operator [FIL_785] (rows=575995635 width=88) predicate:ss_sold_date_sk is not null TableScan [TS_44] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity","ss_list_price"] @@ -453,17 +449,13 @@ Stage-0 <-Map 76 [SIMPLE_EDGE] SHUFFLE [RS_252] PartitionCols:_col0 - Select Operator [SEL_250] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_813] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Map 89 [SIMPLE_EDGE] SHUFFLE [RS_251] PartitionCols:_col0 - Select Operator [SEL_247] (rows=287989836 width=135) + Select Operator [SEL_56] (rows=287989836 width=135) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_812] (rows=287989836 width=135) + Filter Operator [FIL_787] (rows=287989836 width=135) predicate:cs_sold_date_sk is not null TableScan [TS_54] (rows=287989836 width=135) default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity","cs_list_price"] @@ -480,17 +472,13 @@ Stage-0 <-Map 84 [SIMPLE_EDGE] SHUFFLE [RS_263] PartitionCols:_col0 - Select Operator [SEL_261] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_815] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - Please refer to the previous TableScan [TS_24] + Please refer to the previous Select Operator [SEL_26] <-Map 90 [SIMPLE_EDGE] SHUFFLE [RS_262] PartitionCols:_col0 - Select Operator [SEL_258] (rows=144002668 width=135) + Select Operator [SEL_67] (rows=144002668 width=135) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_814] (rows=144002668 width=135) + Filter Operator [FIL_789] (rows=144002668 width=135) predicate:ws_sold_date_sk is not null TableScan [TS_65] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity","ws_list_price"] @@ -578,17 +566,13 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_294] PartitionCols:_col0 - Select Operator [SEL_289] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_821] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 97 [SIMPLE_EDGE] SHUFFLE [RS_293] PartitionCols:_col0 - Select Operator [SEL_286] (rows=575995635 width=88) + Select Operator [SEL_95] (rows=575995635 width=88) Output:["_col0","_col1"] - Filter Operator [FIL_820] (rows=575995635 width=88) + Filter Operator [FIL_795] (rows=575995635 width=88) predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) TableScan [TS_93] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] @@ -624,17 +608,13 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_315] PartitionCols:_col0 - Select Operator [SEL_310] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_824] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 98 [SIMPLE_EDGE] SHUFFLE [RS_314] PartitionCols:_col0 - Select Operator [SEL_307] (rows=287989836 width=135) + Select Operator [SEL_116] (rows=287989836 width=135) Output:["_col0","_col1"] - Filter Operator [FIL_823] (rows=287989836 width=135) + Filter Operator [FIL_798] (rows=287989836 width=135) predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) TableScan [TS_114] (rows=287989836 width=135) default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk"] @@ -670,17 +650,13 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_337] PartitionCols:_col0 - Select Operator [SEL_332] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_827] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 99 [SIMPLE_EDGE] SHUFFLE [RS_336] PartitionCols:_col0 - Select Operator [SEL_329] (rows=144002668 width=135) + Select Operator [SEL_138] (rows=144002668 width=135) Output:["_col0","_col1"] - Filter Operator [FIL_826] (rows=144002668 width=135) + Filter Operator [FIL_801] (rows=144002668 width=135) predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) TableScan [TS_136] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk"] @@ -692,9 +668,9 @@ Stage-0 <-Map 93 [SIMPLE_EDGE] SHUFFLE [RS_364] PartitionCols:_col0 - Select Operator [SEL_277] (rows=18262 width=1119) + Select Operator [SEL_86] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_817] (rows=18262 width=1119) + Filter Operator [FIL_792] (rows=18262 width=1119) predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null) TableScan [TS_84] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] @@ -742,19 +718,11 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_390] PartitionCols:_col0 - Select Operator [SEL_388] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_830] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_389] PartitionCols:_col0 - Select Operator [SEL_385] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_829] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 75 [CONTAINS] Reduce Output Operator [RS_416] Group By Operator [GBY_415] (rows=1 width=8) @@ -768,19 +736,11 @@ Stage-0 <-Map 76 [SIMPLE_EDGE] SHUFFLE [RS_400] PartitionCols:_col0 - Select Operator [SEL_398] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_832] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Map 72 [SIMPLE_EDGE] SHUFFLE [RS_399] PartitionCols:_col0 - Select Operator [SEL_395] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_831] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - Please refer to the previous TableScan [TS_10] + Please refer to the previous Select Operator [SEL_12] <-Reducer 83 [CONTAINS] Reduce Output Operator [RS_416] Group By Operator [GBY_415] (rows=1 width=8) @@ -794,19 +754,11 @@ Stage-0 <-Map 84 [SIMPLE_EDGE] SHUFFLE [RS_411] PartitionCols:_col0 - Select Operator [SEL_409] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_834] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - Please refer to the previous TableScan [TS_24] + Please refer to the previous Select Operator [SEL_26] <-Map 80 [SIMPLE_EDGE] SHUFFLE [RS_410] PartitionCols:_col0 - Select Operator [SEL_406] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_833] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - Please refer to the previous TableScan [TS_21] + Please refer to the previous Select Operator [SEL_23] <-Reducer 56 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_569] Group By Operator [GBY_462] (rows=1 width=288) @@ -825,19 +777,11 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_434] PartitionCols:_col0 - Select Operator [SEL_432] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_836] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 88 [SIMPLE_EDGE] SHUFFLE [RS_433] PartitionCols:_col0 - Select Operator [SEL_429] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_835] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - Please refer to the previous TableScan [TS_44] + Please refer to the previous Select Operator [SEL_46] <-Reducer 79 [CONTAINS] Reduce Output Operator [RS_461] Group By Operator [GBY_460] (rows=1 width=288) @@ -851,19 +795,11 @@ Stage-0 <-Map 76 [SIMPLE_EDGE] SHUFFLE [RS_444] PartitionCols:_col0 - Select Operator [SEL_442] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_838] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Map 89 [SIMPLE_EDGE] SHUFFLE [RS_443] PartitionCols:_col0 - Select Operator [SEL_439] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_837] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - Please refer to the previous TableScan [TS_54] + Please refer to the previous Select Operator [SEL_56] <-Reducer 87 [CONTAINS] Reduce Output Operator [RS_461] Group By Operator [GBY_460] (rows=1 width=288) @@ -877,19 +813,11 @@ Stage-0 <-Map 84 [SIMPLE_EDGE] SHUFFLE [RS_455] PartitionCols:_col0 - Select Operator [SEL_453] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_840] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - Please refer to the previous TableScan [TS_24] + Please refer to the previous Select Operator [SEL_26] <-Map 90 [SIMPLE_EDGE] SHUFFLE [RS_454] PartitionCols:_col0 - Select Operator [SEL_450] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_839] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - Please refer to the previous TableScan [TS_65] + Please refer to the previous Select Operator [SEL_67] <-Reducer 65 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_570] Group By Operator [GBY_566] (rows=174243235 width=135) @@ -973,19 +901,11 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_486] PartitionCols:_col0 - Select Operator [SEL_481] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_846] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 97 [SIMPLE_EDGE] SHUFFLE [RS_485] PartitionCols:_col0 - Select Operator [SEL_478] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_845] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - Please refer to the previous TableScan [TS_93] + Please refer to the previous Select Operator [SEL_95] <-Reducer 68 [CONTAINS] Reduce Output Operator [RS_543] PartitionCols:_col0, _col1, _col2 @@ -1018,19 +938,11 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_507] PartitionCols:_col0 - Select Operator [SEL_502] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_849] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 98 [SIMPLE_EDGE] SHUFFLE [RS_506] PartitionCols:_col0 - Select Operator [SEL_499] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_848] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - Please refer to the previous TableScan [TS_114] + Please refer to the previous Select Operator [SEL_116] <-Reducer 71 [CONTAINS] Reduce Output Operator [RS_543] PartitionCols:_col0, _col1, _col2 @@ -1063,19 +975,11 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_529] PartitionCols:_col0 - Select Operator [SEL_524] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_852] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 99 [SIMPLE_EDGE] SHUFFLE [RS_528] PartitionCols:_col0 - Select Operator [SEL_521] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_851] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - Please refer to the previous TableScan [TS_136] + Please refer to the previous Select Operator [SEL_138] <-Reducer 95 [SIMPLE_EDGE] SHUFFLE [RS_558] PartitionCols:_col1 @@ -1084,11 +988,7 @@ Stage-0 <-Map 93 [SIMPLE_EDGE] SHUFFLE [RS_556] PartitionCols:_col0 - Select Operator [SEL_469] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_842] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null) - Please refer to the previous TableScan [TS_84] + Please refer to the previous Select Operator [SEL_86] <-Map 101 [SIMPLE_EDGE] SHUFFLE [RS_555] PartitionCols:_col0 @@ -1127,19 +1027,11 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_51] PartitionCols:_col0 - Select Operator [SEL_49] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_786] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 88 [SIMPLE_EDGE] SHUFFLE [RS_50] PartitionCols:_col0 - Select Operator [SEL_46] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_785] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - Please refer to the previous TableScan [TS_44] + Please refer to the previous Select Operator [SEL_46] <-Reducer 77 [CONTAINS] Reduce Output Operator [RS_78] Group By Operator [GBY_77] (rows=1 width=288) @@ -1153,19 +1045,11 @@ Stage-0 <-Map 76 [SIMPLE_EDGE] SHUFFLE [RS_61] PartitionCols:_col0 - Select Operator [SEL_59] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_788] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Map 89 [SIMPLE_EDGE] SHUFFLE [RS_60] PartitionCols:_col0 - Select Operator [SEL_56] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_787] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - Please refer to the previous TableScan [TS_54] + Please refer to the previous Select Operator [SEL_56] <-Reducer 85 [CONTAINS] Reduce Output Operator [RS_78] Group By Operator [GBY_77] (rows=1 width=288) @@ -1179,19 +1063,11 @@ Stage-0 <-Map 84 [SIMPLE_EDGE] SHUFFLE [RS_72] PartitionCols:_col0 - Select Operator [SEL_70] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_790] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - Please refer to the previous TableScan [TS_24] + Please refer to the previous Select Operator [SEL_26] <-Map 90 [SIMPLE_EDGE] SHUFFLE [RS_71] PartitionCols:_col0 - Select Operator [SEL_67] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_789] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - Please refer to the previous TableScan [TS_65] + Please refer to the previous Select Operator [SEL_67] <-Reducer 29 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_187] Group By Operator [GBY_183] (rows=696954748 width=88) @@ -1275,19 +1151,11 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_103] PartitionCols:_col0 - Select Operator [SEL_98] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_796] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 97 [SIMPLE_EDGE] SHUFFLE [RS_102] PartitionCols:_col0 - Select Operator [SEL_95] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_795] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - Please refer to the previous TableScan [TS_93] + Please refer to the previous Select Operator [SEL_95] <-Reducer 32 [CONTAINS] Reduce Output Operator [RS_160] PartitionCols:_col0, _col1, _col2 @@ -1320,19 +1188,11 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_124] PartitionCols:_col0 - Select Operator [SEL_119] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_799] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 98 [SIMPLE_EDGE] SHUFFLE [RS_123] PartitionCols:_col0 - Select Operator [SEL_116] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_798] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - Please refer to the previous TableScan [TS_114] + Please refer to the previous Select Operator [SEL_116] <-Reducer 35 [CONTAINS] Reduce Output Operator [RS_160] PartitionCols:_col0, _col1, _col2 @@ -1365,19 +1225,11 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_146] PartitionCols:_col0 - Select Operator [SEL_141] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_802] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 99 [SIMPLE_EDGE] SHUFFLE [RS_145] PartitionCols:_col0 - Select Operator [SEL_138] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_801] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - Please refer to the previous TableScan [TS_136] + Please refer to the previous Select Operator [SEL_138] <-Reducer 92 [SIMPLE_EDGE] SHUFFLE [RS_175] PartitionCols:_col1 @@ -1386,11 +1238,7 @@ Stage-0 <-Map 93 [SIMPLE_EDGE] SHUFFLE [RS_173] PartitionCols:_col0 - Select Operator [SEL_86] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_792] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null) - Please refer to the previous TableScan [TS_84] + Please refer to the previous Select Operator [SEL_86] <-Map 91 [SIMPLE_EDGE] SHUFFLE [RS_172] PartitionCols:_col0 @@ -1424,19 +1272,11 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_7] PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_780] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_6] PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_779] (rows=575995635 width=88) - predicate:ss_sold_date_sk is not null - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 73 [CONTAINS] Reduce Output Operator [RS_33] Group By Operator [GBY_32] (rows=1 width=8) @@ -1450,19 +1290,11 @@ Stage-0 <-Map 76 [SIMPLE_EDGE] SHUFFLE [RS_17] PartitionCols:_col0 - Select Operator [SEL_15] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_782] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Map 72 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_12] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_781] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - Please refer to the previous TableScan [TS_10] + Please refer to the previous Select Operator [SEL_12] <-Reducer 81 [CONTAINS] Reduce Output Operator [RS_33] Group By Operator [GBY_32] (rows=1 width=8) @@ -1476,17 +1308,9 @@ Stage-0 <-Map 84 [SIMPLE_EDGE] SHUFFLE [RS_28] PartitionCols:_col0 - Select Operator [SEL_26] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_784] (rows=8116 width=1119) - predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null) - Please refer to the previous TableScan [TS_24] + Please refer to the previous Select Operator [SEL_26] <-Map 80 [SIMPLE_EDGE] SHUFFLE [RS_27] PartitionCols:_col0 - Select Operator [SEL_23] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_783] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - Please refer to the previous TableScan [TS_21] + Please refer to the previous Select Operator [SEL_23] diff --git ql/src/test/results/clientpositive/perf/query16.q.out ql/src/test/results/clientpositive/perf/query16.q.out index a7f93f9..239f2b7 100644 --- ql/src/test/results/clientpositive/perf/query16.q.out +++ ql/src/test/results/clientpositive/perf/query16.q.out @@ -150,9 +150,7 @@ Stage-0 Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"] <-Map 15 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_32] - Select Operator [SEL_28] (rows=287989836 width=135) - Output:["_col0","_col1"] - Please refer to the previous TableScan [TS_19] + Please refer to the previous Select Operator [SEL_20] <-Map 20 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_29] Select Operator [SEL_22] (rows=73049 width=4) diff --git ql/src/test/results/clientpositive/perf/query2.q.out ql/src/test/results/clientpositive/perf/query2.q.out index 50d7f7b..1686f57 100644 --- ql/src/test/results/clientpositive/perf/query2.q.out +++ ql/src/test/results/clientpositive/perf/query2.q.out @@ -177,9 +177,9 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_39] PartitionCols:_col0 - Select Operator [SEL_37] (rows=73049 width=1119) + Select Operator [SEL_10] (rows=73049 width=1119) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_89] (rows=73049 width=1119) + Filter Operator [FIL_85] (rows=73049 width=1119) predicate:(d_date_sk is not null and d_week_seq is not null) TableScan [TS_8] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_week_seq","d_day_name"] @@ -232,11 +232,7 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col0 - Select Operator [SEL_10] (rows=73049 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_85] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_week_seq is not null) - Please refer to the previous TableScan [TS_8] + Please refer to the previous Select Operator [SEL_10] <-Union 2 [SIMPLE_EDGE] <-Map 1 [CONTAINS] Reduce Output Operator [RS_11] diff --git ql/src/test/results/clientpositive/perf/query23.q.out ql/src/test/results/clientpositive/perf/query23.q.out index b8cdad3..ebd2271 100644 --- ql/src/test/results/clientpositive/perf/query23.q.out +++ ql/src/test/results/clientpositive/perf/query23.q.out @@ -195,9 +195,9 @@ Stage-0 <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_165] PartitionCols:_col0 - Select Operator [SEL_160] (rows=80000000 width=860) + Select Operator [SEL_41] (rows=80000000 width=860) Output:["_col0"] - Filter Operator [FIL_341] (rows=80000000 width=860) + Filter Operator [FIL_328] (rows=80000000 width=860) predicate:c_customer_sk is not null TableScan [TS_39] (rows=80000000 width=860) default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] @@ -209,18 +209,18 @@ Stage-0 <-Map 31 [SIMPLE_EDGE] SHUFFLE [RS_162] PartitionCols:_col0 - Select Operator [SEL_157] (rows=36525 width=1119) + Select Operator [SEL_38] (rows=36525 width=1119) Output:["_col0"] - Filter Operator [FIL_340] (rows=36525 width=1119) + Filter Operator [FIL_327] (rows=36525 width=1119) predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) TableScan [TS_36] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_161] PartitionCols:_col0 - Select Operator [SEL_154] (rows=575995635 width=88) + Select Operator [SEL_35] (rows=575995635 width=88) Output:["_col0","_col1"] - Filter Operator [FIL_339] (rows=575995635 width=88) + Filter Operator [FIL_326] (rows=575995635 width=88) predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) TableScan [TS_33] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] @@ -248,11 +248,7 @@ Stage-0 <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_194] PartitionCols:_col0 - Select Operator [SEL_189] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_344] (rows=80000000 width=860) - predicate:c_customer_sk is not null - Please refer to the previous TableScan [TS_39] + Please refer to the previous Select Operator [SEL_41] <-Reducer 36 [SIMPLE_EDGE] SHUFFLE [RS_193] PartitionCols:_col1 @@ -261,17 +257,13 @@ Stage-0 <-Map 31 [SIMPLE_EDGE] SHUFFLE [RS_191] PartitionCols:_col0 - Select Operator [SEL_186] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_343] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - Please refer to the previous TableScan [TS_36] + Please refer to the previous Select Operator [SEL_38] <-Map 45 [SIMPLE_EDGE] SHUFFLE [RS_190] PartitionCols:_col0 - Select Operator [SEL_183] (rows=575995635 width=88) + Select Operator [SEL_64] (rows=575995635 width=88) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_342] (rows=575995635 width=88) + Filter Operator [FIL_329] (rows=575995635 width=88) predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) TableScan [TS_62] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_quantity","ss_sales_price"] @@ -291,17 +283,13 @@ Stage-0 <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_214] PartitionCols:_col0 - Select Operator [SEL_212] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_346] (rows=80000000 width=860) - predicate:c_customer_sk is not null - Please refer to the previous TableScan [TS_39] + Please refer to the previous Select Operator [SEL_41] <-Map 46 [SIMPLE_EDGE] SHUFFLE [RS_213] PartitionCols:_col0 - Select Operator [SEL_209] (rows=575995635 width=88) + Select Operator [SEL_90] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_345] (rows=575995635 width=88) + Filter Operator [FIL_332] (rows=575995635 width=88) predicate:ss_customer_sk is not null TableScan [TS_88] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_customer_sk","ss_quantity","ss_sales_price"] @@ -335,9 +323,9 @@ Stage-0 <-Map 19 [SIMPLE_EDGE] SHUFFLE [RS_138] PartitionCols:_col0 - Select Operator [SEL_133] (rows=462000 width=1436) + Select Operator [SEL_14] (rows=462000 width=1436) Output:["_col0","_col1"] - Filter Operator [FIL_338] (rows=462000 width=1436) + Filter Operator [FIL_325] (rows=462000 width=1436) predicate:i_item_sk is not null TableScan [TS_12] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"] @@ -349,18 +337,18 @@ Stage-0 <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_134] PartitionCols:_col0 - Select Operator [SEL_127] (rows=575995635 width=88) + Select Operator [SEL_8] (rows=575995635 width=88) Output:["_col0","_col1"] - Filter Operator [FIL_336] (rows=575995635 width=88) + Filter Operator [FIL_323] (rows=575995635 width=88) predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) TableScan [TS_6] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_135] PartitionCols:_col0 - Select Operator [SEL_130] (rows=36525 width=1119) + Select Operator [SEL_11] (rows=36525 width=1119) Output:["_col0","_col1"] - Filter Operator [FIL_337] (rows=36525 width=1119) + Filter Operator [FIL_324] (rows=36525 width=1119) predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) TableScan [TS_9] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_year"] @@ -372,9 +360,9 @@ Stage-0 <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_229] PartitionCols:_col0 - Select Operator [SEL_124] (rows=18262 width=1119) + Select Operator [SEL_5] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_335] (rows=18262 width=1119) + Filter Operator [FIL_322] (rows=18262 width=1119) predicate:((d_year = 1999) and (d_moy = 1) and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] @@ -430,11 +418,7 @@ Stage-0 <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_46] PartitionCols:_col0 - Select Operator [SEL_41] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_328] (rows=80000000 width=860) - predicate:c_customer_sk is not null - Please refer to the previous TableScan [TS_39] + Please refer to the previous Select Operator [SEL_41] <-Reducer 21 [SIMPLE_EDGE] SHUFFLE [RS_45] PartitionCols:_col1 @@ -443,19 +427,11 @@ Stage-0 <-Map 31 [SIMPLE_EDGE] SHUFFLE [RS_43] PartitionCols:_col0 - Select Operator [SEL_38] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_327] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - Please refer to the previous TableScan [TS_36] + Please refer to the previous Select Operator [SEL_38] <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_42] PartitionCols:_col0 - Select Operator [SEL_35] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_326] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - Please refer to the previous TableScan [TS_33] + Please refer to the previous Select Operator [SEL_35] <-Reducer 35 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_104] Group By Operator [GBY_86] (rows=1 width=224) @@ -480,11 +456,7 @@ Stage-0 <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_75] PartitionCols:_col0 - Select Operator [SEL_70] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_331] (rows=80000000 width=860) - predicate:c_customer_sk is not null - Please refer to the previous TableScan [TS_39] + Please refer to the previous Select Operator [SEL_41] <-Reducer 32 [SIMPLE_EDGE] SHUFFLE [RS_74] PartitionCols:_col1 @@ -493,19 +465,11 @@ Stage-0 <-Map 31 [SIMPLE_EDGE] SHUFFLE [RS_72] PartitionCols:_col0 - Select Operator [SEL_67] (rows=36525 width=1119) - Output:["_col0"] - Filter Operator [FIL_330] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - Please refer to the previous TableScan [TS_36] + Please refer to the previous Select Operator [SEL_38] <-Map 45 [SIMPLE_EDGE] SHUFFLE [RS_71] PartitionCols:_col0 - Select Operator [SEL_64] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_329] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - Please refer to the previous TableScan [TS_62] + Please refer to the previous Select Operator [SEL_64] <-Reducer 42 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_105] Group By Operator [GBY_101] (rows=316797606 width=88) @@ -522,19 +486,11 @@ Stage-0 <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_95] PartitionCols:_col0 - Select Operator [SEL_93] (rows=80000000 width=860) - Output:["_col0"] - Filter Operator [FIL_333] (rows=80000000 width=860) - predicate:c_customer_sk is not null - Please refer to the previous TableScan [TS_39] + Please refer to the previous Select Operator [SEL_41] <-Map 46 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0 - Select Operator [SEL_90] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_332] (rows=575995635 width=88) - predicate:ss_customer_sk is not null - Please refer to the previous TableScan [TS_88] + Please refer to the previous Select Operator [SEL_90] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_115] PartitionCols:_col1 @@ -565,11 +521,7 @@ Stage-0 <-Map 19 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 - Select Operator [SEL_14] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_325] (rows=462000 width=1436) - predicate:i_item_sk is not null - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col1 @@ -578,19 +530,11 @@ Stage-0 <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col0 - Select Operator [SEL_8] (rows=575995635 width=88) - Output:["_col0","_col1"] - Filter Operator [FIL_323] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_item_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_11] (rows=36525 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_324] (rows=36525 width=1119) - predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_112] PartitionCols:_col2 @@ -599,11 +543,7 @@ Stage-0 <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_110] PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_322] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 1) and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_109] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query24.q.out ql/src/test/results/clientpositive/perf/query24.q.out index 2aa0c19..bfc89e2 100644 --- ql/src/test/results/clientpositive/perf/query24.q.out +++ ql/src/test/results/clientpositive/perf/query24.q.out @@ -151,9 +151,9 @@ Stage-0 <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_74] PartitionCols:_col1, upper(_col2) - Select Operator [SEL_60] (rows=40000000 width=1014) + Select Operator [SEL_17] (rows=40000000 width=1014) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_143] (rows=40000000 width=1014) + Filter Operator [FIL_137] (rows=40000000 width=1014) predicate:(ca_zip is not null and ca_country is not null) TableScan [TS_15] (rows=40000000 width=1014) default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_state","ca_zip","ca_country"] @@ -179,9 +179,9 @@ Stage-0 <-Map 19 [SIMPLE_EDGE] SHUFFLE [RS_68] PartitionCols:_col0 - Select Operator [SEL_54] (rows=80000000 width=860) + Select Operator [SEL_14] (rows=80000000 width=860) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_141] (rows=80000000 width=860) + Filter Operator [FIL_136] (rows=80000000 width=860) predicate:(c_customer_sk is not null and c_birth_country is not null) TableScan [TS_12] (rows=80000000 width=860) default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name","c_birth_country"] @@ -193,9 +193,9 @@ Stage-0 <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_65] PartitionCols:_col0 - Select Operator [SEL_51] (rows=852 width=1910) + Select Operator [SEL_11] (rows=852 width=1910) Output:["_col0","_col1","_col3","_col4"] - Filter Operator [FIL_140] (rows=852 width=1910) + Filter Operator [FIL_135] (rows=852 width=1910) predicate:((s_market_id = 7) and s_store_sk is not null and s_zip is not null) TableScan [TS_9] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_market_id","s_state","s_zip"] @@ -207,18 +207,18 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_61] PartitionCols:_col0, _col3 - Select Operator [SEL_45] (rows=575995635 width=88) + Select Operator [SEL_2] (rows=575995635 width=88) Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_138] (rows=575995635 width=88) + Filter Operator [FIL_132] (rows=575995635 width=88) predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_store_sk is not null and ss_customer_sk is not null) TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_sales_price"] <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_62] PartitionCols:_col0, _col1 - Select Operator [SEL_48] (rows=57591150 width=77) + Select Operator [SEL_5] (rows=57591150 width=77) Output:["_col0","_col1"] - Filter Operator [FIL_139] (rows=57591150 width=77) + Filter Operator [FIL_133] (rows=57591150 width=77) predicate:(sr_item_sk is not null and sr_ticket_number is not null) TableScan [TS_3] (rows=57591150 width=77) default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] @@ -242,11 +242,7 @@ Stage-0 <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_31] PartitionCols:_col1, upper(_col2) - Select Operator [SEL_17] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_137] (rows=40000000 width=1014) - predicate:(ca_zip is not null and ca_country is not null) - Please refer to the previous TableScan [TS_15] + Please refer to the previous Select Operator [SEL_17] <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col17, _col21 @@ -255,11 +251,7 @@ Stage-0 <-Map 19 [SIMPLE_EDGE] SHUFFLE [RS_28] PartitionCols:_col0 - Select Operator [SEL_14] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_136] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_birth_country is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_27] PartitionCols:_col1 @@ -268,11 +260,7 @@ Stage-0 <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_25] PartitionCols:_col0 - Select Operator [SEL_11] (rows=852 width=1910) - Output:["_col0","_col1","_col3","_col4"] - Filter Operator [FIL_135] (rows=852 width=1910) - predicate:((s_market_id = 7) and s_store_sk is not null and s_zip is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_24] PartitionCols:_col2 @@ -294,17 +282,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col0, _col3 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_132] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_store_sk is not null and ss_customer_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0, _col1 - Select Operator [SEL_5] (rows=57591150 width=77) - Output:["_col0","_col1"] - Filter Operator [FIL_133] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_ticket_number is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] diff --git ql/src/test/results/clientpositive/perf/query30.q.out ql/src/test/results/clientpositive/perf/query30.q.out index a86f11d..29f2061 100644 --- ql/src/test/results/clientpositive/perf/query30.q.out +++ ql/src/test/results/clientpositive/perf/query30.q.out @@ -163,9 +163,9 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_36] PartitionCols:_col0 - Select Operator [SEL_31] (rows=36524 width=1119) + Select Operator [SEL_11] (rows=36524 width=1119) Output:["_col0"] - Filter Operator [FIL_97] (rows=36524 width=1119) + Filter Operator [FIL_94] (rows=36524 width=1119) predicate:((d_year = 2002) and d_date_sk is not null) TableScan [TS_9] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] @@ -207,9 +207,5 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_11] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_94] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] diff --git ql/src/test/results/clientpositive/perf/query31.q.out ql/src/test/results/clientpositive/perf/query31.q.out index b3a74f3..79b9e52 100644 --- ql/src/test/results/clientpositive/perf/query31.q.out +++ ql/src/test/results/clientpositive/perf/query31.q.out @@ -151,9 +151,9 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_53] PartitionCols:_col0 - Select Operator [SEL_48] (rows=40000000 width=1014) + Select Operator [SEL_8] (rows=40000000 width=1014) Output:["_col0","_col1"] - Filter Operator [FIL_244] (rows=40000000 width=1014) + Filter Operator [FIL_238] (rows=40000000 width=1014) predicate:(ca_address_sk is not null and ca_county is not null) TableScan [TS_6] (rows=40000000 width=1014) default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"] @@ -174,9 +174,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_49] PartitionCols:_col0 - Select Operator [SEL_42] (rows=575995635 width=88) + Select Operator [SEL_2] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_242] (rows=575995635 width=88) + Filter Operator [FIL_236] (rows=575995635 width=88) predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"] @@ -200,11 +200,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_73] PartitionCols:_col0 - Select Operator [SEL_68] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_247] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 13 [SIMPLE_EDGE] SHUFFLE [RS_72] PartitionCols:_col1 @@ -242,11 +238,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_93] PartitionCols:_col0 - Select Operator [SEL_88] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_250] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 17 [SIMPLE_EDGE] SHUFFLE [RS_92] PartitionCols:_col1 @@ -263,11 +255,7 @@ Stage-0 <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_89] PartitionCols:_col0 - Select Operator [SEL_82] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_248] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - Please refer to the previous TableScan [TS_60] + Please refer to the previous Select Operator [SEL_62] <-Reducer 22 [SIMPLE_EDGE] SHUFFLE [RS_122] PartitionCols:_col0 @@ -283,11 +271,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_113] PartitionCols:_col0 - Select Operator [SEL_108] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_253] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_112] PartitionCols:_col1 @@ -304,11 +288,7 @@ Stage-0 <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_109] PartitionCols:_col0 - Select Operator [SEL_102] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_251] (rows=144002668 width=135) - predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) - Please refer to the previous TableScan [TS_60] + Please refer to the previous Select Operator [SEL_62] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_125] PartitionCols:_col0 @@ -324,11 +304,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_8] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_238] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col1 @@ -345,11 +321,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_9] PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_236] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 8 [SIMPLE_EDGE] SHUFFLE [RS_126] PartitionCols:_col0 @@ -365,11 +337,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_33] PartitionCols:_col0 - Select Operator [SEL_28] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_241] (rows=40000000 width=1014) - predicate:(ca_address_sk is not null and ca_county is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_32] PartitionCols:_col1 @@ -386,9 +354,5 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_29] PartitionCols:_col0 - Select Operator [SEL_22] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_239] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] diff --git ql/src/test/results/clientpositive/perf/query32.q.out ql/src/test/results/clientpositive/perf/query32.q.out index 66b8485..10ad060 100644 --- ql/src/test/results/clientpositive/perf/query32.q.out +++ ql/src/test/results/clientpositive/perf/query32.q.out @@ -136,17 +136,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col0 - Select Operator [SEL_8] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_55] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 8 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_11] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_56] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-03-18 00:00:00.0 AND 1998-06-16 01:00:00.0 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] diff --git ql/src/test/results/clientpositive/perf/query33.q.out ql/src/test/results/clientpositive/perf/query33.q.out index 2058830..aab3a19 100644 --- ql/src/test/results/clientpositive/perf/query33.q.out +++ ql/src/test/results/clientpositive/perf/query33.q.out @@ -209,9 +209,9 @@ Stage-0 <-Map 26 [SIMPLE_EDGE] SHUFFLE [RS_60] PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) + Select Operator [SEL_18] (rows=20000000 width=1014) Output:["_col0"] - Filter Operator [FIL_167] (rows=20000000 width=1014) + Filter Operator [FIL_162] (rows=20000000 width=1014) predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) TableScan [TS_16] (rows=40000000 width=1014) default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] @@ -223,9 +223,9 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_57] PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) + Select Operator [SEL_15] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_166] (rows=18262 width=1119) + Filter Operator [FIL_161] (rows=18262 width=1119) predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) TableScan [TS_13] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] @@ -246,9 +246,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_63] PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) + Select Operator [SEL_2] (rows=462000 width=1436) Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=462000 width=1436) + Filter Operator [FIL_158] (rows=462000 width=1436) predicate:(i_manufact_id is not null and i_item_sk is not null) TableScan [TS_0] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] @@ -260,11 +260,11 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_44] PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) + Group By Operator [GBY_6] (rows=231000 width=1436) Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_42] (rows=231000 width=1436) + Select Operator [SEL_5] (rows=231000 width=1436) Output:["i_manufact_id"] - Filter Operator [FIL_164] (rows=231000 width=1436) + Filter Operator [FIL_159] (rows=231000 width=1436) predicate:((i_category) IN ('Books') and i_manufact_id is not null) TableScan [TS_3] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"] @@ -290,11 +290,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_101] PartitionCols:_col1 - Select Operator [SEL_77] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_168] (rows=462000 width=1436) - predicate:(i_manufact_id is not null and i_item_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 17 [SIMPLE_EDGE] SHUFFLE [RS_102] PartitionCols:_col0 @@ -303,13 +299,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_82] PartitionCols:_col0 - Group By Operator [GBY_81] (rows=231000 width=1436) - Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_80] (rows=231000 width=1436) - Output:["i_manufact_id"] - Filter Operator [FIL_169] (rows=231000 width=1436) - predicate:((i_category) IN ('Books') and i_manufact_id is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Group By Operator [GBY_6] <-Reducer 25 [SIMPLE_EDGE] SHUFFLE [RS_105] PartitionCols:_col3 @@ -320,11 +310,7 @@ Stage-0 <-Map 26 [SIMPLE_EDGE] SHUFFLE [RS_98] PartitionCols:_col0 - Select Operator [SEL_93] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_172] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - Please refer to the previous TableScan [TS_16] + Please refer to the previous Select Operator [SEL_18] <-Reducer 24 [SIMPLE_EDGE] SHUFFLE [RS_97] PartitionCols:_col2 @@ -333,11 +319,7 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_95] PartitionCols:_col0 - Select Operator [SEL_90] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_171] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Map 28 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0 @@ -369,11 +351,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_26] PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=462000 width=1436) - predicate:(i_manufact_id is not null and i_item_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_27] PartitionCols:_col0 @@ -382,13 +360,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_7] PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_manufact_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_manufact_id"] - Filter Operator [FIL_159] (rows=231000 width=1436) - predicate:((i_category) IN ('Books') and i_manufact_id is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Group By Operator [GBY_6] <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col3 @@ -399,11 +371,7 @@ Stage-0 <-Map 26 [SIMPLE_EDGE] SHUFFLE [RS_23] PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_162] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - Please refer to the previous TableScan [TS_16] + Please refer to the previous Select Operator [SEL_18] <-Reducer 19 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col2 @@ -412,11 +380,7 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_20] PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_161] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query35.q.out ql/src/test/results/clientpositive/perf/query35.q.out index b286a07..2815257 100644 --- ql/src/test/results/clientpositive/perf/query35.q.out +++ ql/src/test/results/clientpositive/perf/query35.q.out @@ -170,9 +170,9 @@ Stage-0 <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_55] PartitionCols:_col0 - Select Operator [SEL_53] (rows=12174 width=1119) + Select Operator [SEL_14] (rows=12174 width=1119) Output:["_col0"] - Filter Operator [FIL_106] (rows=12174 width=1119) + Filter Operator [FIL_102] (rows=12174 width=1119) predicate:((d_year = 1999) and (d_qoy < 4) and d_date_sk is not null) TableScan [TS_12] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"] @@ -209,11 +209,7 @@ Stage-0 <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_14] (rows=12174 width=1119) - Output:["_col0"] - Filter Operator [FIL_102] (rows=12174 width=1119) - predicate:((d_year = 1999) and (d_qoy < 4) and d_date_sk is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col0 @@ -238,11 +234,7 @@ Stage-0 <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col0 - Select Operator [SEL_28] (rows=12174 width=1119) - Output:["_col0"] - Filter Operator [FIL_104] (rows=12174 width=1119) - predicate:((d_year = 1999) and (d_qoy < 4) and d_date_sk is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_29] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query38.q.out ql/src/test/results/clientpositive/perf/query38.q.out index ae9ada5..1fae4e2 100644 --- ql/src/test/results/clientpositive/perf/query38.q.out +++ ql/src/test/results/clientpositive/perf/query38.q.out @@ -98,9 +98,9 @@ Stage-0 <-Map 15 [SIMPLE_EDGE] SHUFFLE [RS_39] PartitionCols:_col0 - Select Operator [SEL_34] (rows=80000000 width=860) + Select Operator [SEL_8] (rows=80000000 width=860) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_113] (rows=80000000 width=860) + Filter Operator [FIL_110] (rows=80000000 width=860) predicate:c_customer_sk is not null TableScan [TS_6] (rows=80000000 width=860) default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] @@ -112,9 +112,9 @@ Stage-0 <-Map 8 [SIMPLE_EDGE] SHUFFLE [RS_36] PartitionCols:_col0 - Select Operator [SEL_31] (rows=8116 width=1119) + Select Operator [SEL_5] (rows=8116 width=1119) Output:["_col0","_col1"] - Filter Operator [FIL_112] (rows=8116 width=1119) + Filter Operator [FIL_109] (rows=8116 width=1119) predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] @@ -146,11 +146,7 @@ Stage-0 <-Map 15 [SIMPLE_EDGE] SHUFFLE [RS_66] PartitionCols:_col0 - Select Operator [SEL_61] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_116] (rows=80000000 width=860) - predicate:c_customer_sk is not null - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_65] PartitionCols:_col1 @@ -159,11 +155,7 @@ Stage-0 <-Map 8 [SIMPLE_EDGE] SHUFFLE [RS_63] PartitionCols:_col0 - Select Operator [SEL_58] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_115] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_62] PartitionCols:_col0 @@ -192,11 +184,7 @@ Stage-0 <-Map 15 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_110] (rows=80000000 width=860) - predicate:c_customer_sk is not null - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col1 @@ -205,11 +193,7 @@ Stage-0 <-Map 8 [SIMPLE_EDGE] SHUFFLE [RS_10] PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_109] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_9] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query39.q.out ql/src/test/results/clientpositive/perf/query39.q.out index cf139f2..7607ba0 100644 --- ql/src/test/results/clientpositive/perf/query39.q.out +++ ql/src/test/results/clientpositive/perf/query39.q.out @@ -99,9 +99,9 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_47] PartitionCols:_col0 - Select Operator [SEL_39] (rows=27 width=1029) + Select Operator [SEL_11] (rows=27 width=1029) Output:["_col0","_col1"] - Filter Operator [FIL_97] (rows=27 width=1029) + Filter Operator [FIL_93] (rows=27 width=1029) predicate:w_warehouse_sk is not null TableScan [TS_9] (rows=27 width=1029) default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"] @@ -113,9 +113,9 @@ Stage-0 <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_44] PartitionCols:_col0 - Select Operator [SEL_36] (rows=462000 width=1436) + Select Operator [SEL_8] (rows=462000 width=1436) Output:["_col0"] - Filter Operator [FIL_96] (rows=462000 width=1436) + Filter Operator [FIL_92] (rows=462000 width=1436) predicate:i_item_sk is not null TableScan [TS_6] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk"] @@ -127,9 +127,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_40] PartitionCols:_col0 - Select Operator [SEL_30] (rows=37584000 width=15) + Select Operator [SEL_2] (rows=37584000 width=15) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_94] (rows=37584000 width=15) + Filter Operator [FIL_90] (rows=37584000 width=15) predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null) TableScan [TS_0] (rows=37584000 width=15) default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"] @@ -163,11 +163,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 - Select Operator [SEL_11] (rows=27 width=1029) - Output:["_col0","_col1"] - Filter Operator [FIL_93] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col2 @@ -176,11 +172,7 @@ Stage-0 <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0"] - Filter Operator [FIL_92] (rows=462000 width=1436) - predicate:i_item_sk is not null - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col1 @@ -189,11 +181,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col0 - Select Operator [SEL_2] (rows=37584000 width=15) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_90] (rows=37584000 width=15) - predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query4.q.out ql/src/test/results/clientpositive/perf/query4.q.out index 5f41b4a..b03a91e 100644 --- ql/src/test/results/clientpositive/perf/query4.q.out +++ ql/src/test/results/clientpositive/perf/query4.q.out @@ -277,9 +277,9 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_34] PartitionCols:_col0 - Select Operator [SEL_29] (rows=80000000 width=860) + Select Operator [SEL_8] (rows=80000000 width=860) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_268] (rows=80000000 width=860) + Filter Operator [FIL_265] (rows=80000000 width=860) predicate:(c_customer_sk is not null and c_customer_id is not null) TableScan [TS_6] (rows=80000000 width=860) default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_first_name","c_last_name","c_preferred_cust_flag","c_birth_country","c_login","c_email_address"] @@ -327,11 +327,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_56] PartitionCols:_col0 - Select Operator [SEL_51] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_271] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_customer_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 14 [SIMPLE_EDGE] SHUFFLE [RS_55] PartitionCols:_col1 @@ -373,11 +369,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_100] PartitionCols:_col0 - Select Operator [SEL_95] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_277] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_customer_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 17 [SIMPLE_EDGE] SHUFFLE [RS_99] PartitionCols:_col1 @@ -394,11 +386,7 @@ Stage-0 <-Map 25 [SIMPLE_EDGE] SHUFFLE [RS_96] PartitionCols:_col0 - Select Operator [SEL_89] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_275] (rows=287989836 width=135) - predicate:(cs_bill_customer_sk is not null and cs_sold_date_sk is not null) - Please refer to the previous TableScan [TS_43] + Please refer to the previous Select Operator [SEL_45] <-Reducer 22 [SIMPLE_EDGE] SHUFFLE [RS_134] PartitionCols:_col0 @@ -418,11 +406,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_121] PartitionCols:_col0 - Select Operator [SEL_116] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_280] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_customer_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_120] PartitionCols:_col1 @@ -439,11 +423,7 @@ Stage-0 <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_117] PartitionCols:_col0 - Select Operator [SEL_110] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_278] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - Please refer to the previous TableScan [TS_21] + Please refer to the previous Select Operator [SEL_23] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_129] PartitionCols:_col0 @@ -463,11 +443,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_265] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_customer_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col1 @@ -511,11 +487,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_78] PartitionCols:_col0 - Select Operator [SEL_73] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Filter Operator [FIL_274] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_customer_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 7 [SIMPLE_EDGE] SHUFFLE [RS_77] PartitionCols:_col1 @@ -532,9 +504,5 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_74] PartitionCols:_col0 - Select Operator [SEL_67] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_272] (rows=144002668 width=135) - predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] diff --git ql/src/test/results/clientpositive/perf/query44.q.out ql/src/test/results/clientpositive/perf/query44.q.out index 5665480..07187b1 100644 --- ql/src/test/results/clientpositive/perf/query44.q.out +++ ql/src/test/results/clientpositive/perf/query44.q.out @@ -200,11 +200,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_90] PartitionCols:_col0 - Select Operator [SEL_47] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_126] (rows=462000 width=1436) - predicate:i_item_sk is not null - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 14 [SIMPLE_EDGE] SHUFFLE [RS_91] PartitionCols:_col0 @@ -240,12 +236,7 @@ Stage-0 <-Map 6 [SIMPLE_EDGE] SHUFFLE [RS_53] PartitionCols:_col0 - Group By Operator [GBY_52] (rows=143998908 width=88) - Output:["_col0"],keys:410 - Select Operator [SEL_50] (rows=143998908 width=88) - Filter Operator [FIL_128] (rows=143998908 width=88) - predicate:((ss_store_sk = 410) and ss_hdemo_sk is null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Group By Operator [GBY_7] <-Reducer 17 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_79] Select Operator [SEL_70] (rows=71999454 width=88) @@ -255,13 +246,7 @@ Stage-0 <-Map 15 [SIMPLE_EDGE] SHUFFLE [RS_68] PartitionCols:_col0 - Group By Operator [GBY_67] (rows=143998908 width=88) - Output:["_col0","_col1"],aggregations:["avg(_col1)"],keys:410 - Select Operator [SEL_65] (rows=143998908 width=88) - Output:["_col1"] - Filter Operator [FIL_129] (rows=143998908 width=88) - predicate:((ss_store_sk = 410) and ss_hdemo_sk is null) - Please refer to the previous TableScan [TS_18] + Please refer to the previous Group By Operator [GBY_22] <-Reducer 20 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_80] Group By Operator [GBY_76] (rows=143998908 width=88) @@ -269,11 +254,5 @@ Stage-0 <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_75] PartitionCols:_col0 - Group By Operator [GBY_74] (rows=287997817 width=88) - Output:["_col0","_col1"],aggregations:["avg(ss_net_profit)"],keys:ss_item_sk - Select Operator [SEL_73] (rows=287997817 width=88) - Output:["ss_item_sk","ss_net_profit"] - Filter Operator [FIL_130] (rows=287997817 width=88) - predicate:(ss_store_sk = 410) - Please refer to the previous TableScan [TS_26] + Please refer to the previous Group By Operator [GBY_29] diff --git ql/src/test/results/clientpositive/perf/query46.q.out ql/src/test/results/clientpositive/perf/query46.q.out index 6806703..380944a 100644 --- ql/src/test/results/clientpositive/perf/query46.q.out +++ ql/src/test/results/clientpositive/perf/query46.q.out @@ -136,11 +136,7 @@ Stage-0 <-Map 5 [SIMPLE_EDGE] SHUFFLE [RS_31] PartitionCols:_col0 - Select Operator [SEL_20] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col3 diff --git ql/src/test/results/clientpositive/perf/query47.q.out ql/src/test/results/clientpositive/perf/query47.q.out index d5e1922..efbd89c 100644 --- ql/src/test/results/clientpositive/perf/query47.q.out +++ ql/src/test/results/clientpositive/perf/query47.q.out @@ -177,9 +177,9 @@ Stage-0 <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_52] PartitionCols:_col0 - Select Operator [SEL_44] (rows=1704 width=1910) + Select Operator [SEL_11] (rows=1704 width=1910) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_174] (rows=1704 width=1910) + Filter Operator [FIL_168] (rows=1704 width=1910) predicate:(s_store_sk is not null and s_store_name is not null and s_company_name is not null) TableScan [TS_9] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_name"] @@ -191,9 +191,9 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_49] PartitionCols:_col0 - Select Operator [SEL_41] (rows=462000 width=1436) + Select Operator [SEL_8] (rows=462000 width=1436) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_173] (rows=462000 width=1436) + Filter Operator [FIL_167] (rows=462000 width=1436) predicate:(i_item_sk is not null and i_category is not null and i_brand is not null) TableScan [TS_6] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"] @@ -205,18 +205,18 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_45] PartitionCols:_col0 - Select Operator [SEL_35] (rows=575995635 width=88) + Select Operator [SEL_2] (rows=575995635 width=88) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_171] (rows=575995635 width=88) + Filter Operator [FIL_165] (rows=575995635 width=88) predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null) TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"] <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_46] PartitionCols:_col0 - Select Operator [SEL_38] (rows=73048 width=1119) + Select Operator [SEL_5] (rows=73048 width=1119) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_172] (rows=73048 width=1119) + Filter Operator [FIL_166] (rows=73048 width=1119) predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] @@ -248,11 +248,7 @@ Stage-0 <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_89] PartitionCols:_col0 - Select Operator [SEL_81] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_179] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_store_name is not null and s_company_name is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 16 [SIMPLE_EDGE] SHUFFLE [RS_88] PartitionCols:_col2 @@ -261,11 +257,7 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_86] PartitionCols:_col0 - Select Operator [SEL_78] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_178] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_category is not null and i_brand is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_85] PartitionCols:_col1 @@ -274,19 +266,11 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_82] PartitionCols:_col0 - Select Operator [SEL_72] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_176] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_83] PartitionCols:_col0 - Select Operator [SEL_75] (rows=73048 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_177] (rows=73048 width=1119) - predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_103] PartitionCols:_col0, _col1, _col2, _col3, (_col7 + 1) @@ -315,11 +299,7 @@ Stage-0 <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 - Select Operator [SEL_11] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_168] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_store_name is not null and s_company_name is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col2 @@ -328,11 +308,7 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_8] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_167] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_category is not null and i_brand is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col1 @@ -341,17 +317,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_165] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_5] (rows=73048 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_166] (rows=73048 width=1119) - predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] diff --git ql/src/test/results/clientpositive/perf/query49.q.out ql/src/test/results/clientpositive/perf/query49.q.out index 8b8ad8b..f8c0649 100644 --- ql/src/test/results/clientpositive/perf/query49.q.out +++ ql/src/test/results/clientpositive/perf/query49.q.out @@ -342,9 +342,9 @@ Stage-0 <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_81] PartitionCols:_col0 - Select Operator [SEL_76] (rows=18262 width=1119) + Select Operator [SEL_5] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_139] (rows=18262 width=1119) + Filter Operator [FIL_131] (rows=18262 width=1119) predicate:((d_year = 2000) and (d_moy = 12) and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] @@ -420,11 +420,7 @@ Stage-0 <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_42] PartitionCols:_col0 - Select Operator [SEL_37] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_135] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 12) and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_41] PartitionCols:_col0 @@ -487,11 +483,7 @@ Stage-0 <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_10] PartitionCols:_col0 - Select Operator [SEL_5] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_131] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 12) and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_9] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query5.q.out ql/src/test/results/clientpositive/perf/query5.q.out index f3f122c..ea072d9 100644 --- ql/src/test/results/clientpositive/perf/query5.q.out +++ ql/src/test/results/clientpositive/perf/query5.q.out @@ -323,9 +323,9 @@ Stage-0 <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_40] PartitionCols:_col0 - Select Operator [SEL_35] (rows=8116 width=1119) + Select Operator [SEL_10] (rows=8116 width=1119) Output:["_col0"] - Filter Operator [FIL_124] (rows=8116 width=1119) + Filter Operator [FIL_120] (rows=8116 width=1119) predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-08-18 00:00:00.0 and d_date_sk is not null) TableScan [TS_8] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] @@ -381,11 +381,7 @@ Stage-0 <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_73] PartitionCols:_col0 - Select Operator [SEL_68] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_129] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-08-18 00:00:00.0 and d_date_sk is not null) - Please refer to the previous TableScan [TS_8] + Please refer to the previous Select Operator [SEL_10] <-Union 23 [SIMPLE_EDGE] <-Map 22 [CONTAINS] Reduce Output Operator [RS_72] @@ -454,11 +450,7 @@ Stage-0 <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col0 - Select Operator [SEL_10] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_120] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-08-18 00:00:00.0 and d_date_sk is not null) - Please refer to the previous TableScan [TS_8] + Please refer to the previous Select Operator [SEL_10] <-Union 2 [SIMPLE_EDGE] <-Map 1 [CONTAINS] Reduce Output Operator [RS_14] diff --git ql/src/test/results/clientpositive/perf/query51.q.out ql/src/test/results/clientpositive/perf/query51.q.out index 0ce3e9f..5f549c5 100644 --- ql/src/test/results/clientpositive/perf/query51.q.out +++ ql/src/test/results/clientpositive/perf/query51.q.out @@ -175,11 +175,7 @@ Stage-0 <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_27] PartitionCols:_col0 - Select Operator [SEL_25] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_62] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_26] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query54.q.out ql/src/test/results/clientpositive/perf/query54.q.out index 3cbcbe3..b9d0b8b 100644 --- ql/src/test/results/clientpositive/perf/query54.q.out +++ ql/src/test/results/clientpositive/perf/query54.q.out @@ -222,7 +222,7 @@ Stage-0 Output:["_col0"],keys:_col0 Select Operator [SEL_65] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_179] (rows=18262 width=1119) + Filter Operator [FIL_169] (rows=18262 width=1119) predicate:((d_year = 1999) and (d_moy = 3)) TableScan [TS_0] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"] @@ -348,9 +348,7 @@ Stage-0 Output:["_col0"],keys:_col0 Select Operator [SEL_83] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_181] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3)) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Filter Operator [FIL_169] <-Reducer 3 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_111] Merge Join Operator [MERGEJOIN_191] (rows=9131 width=1128) @@ -366,9 +364,7 @@ Stage-0 Output:["_col0"],keys:_col0 Select Operator [SEL_2] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_169] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3)) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Filter Operator [FIL_169] <-Reducer 9 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_109] Select Operator [SEL_22] (rows=1 width=8) @@ -386,11 +382,5 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Group By Operator [GBY_12] (rows=18262 width=1119) - Output:["_col0"],keys:_col0 - Select Operator [SEL_10] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_170] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 3)) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Group By Operator [GBY_4] diff --git ql/src/test/results/clientpositive/perf/query56.q.out ql/src/test/results/clientpositive/perf/query56.q.out index 4c11d01..8f4606a 100644 --- ql/src/test/results/clientpositive/perf/query56.q.out +++ ql/src/test/results/clientpositive/perf/query56.q.out @@ -195,9 +195,9 @@ Stage-0 <-Map 26 [SIMPLE_EDGE] SHUFFLE [RS_60] PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) + Select Operator [SEL_18] (rows=20000000 width=1014) Output:["_col0"] - Filter Operator [FIL_167] (rows=20000000 width=1014) + Filter Operator [FIL_162] (rows=20000000 width=1014) predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) TableScan [TS_16] (rows=40000000 width=1014) default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] @@ -209,9 +209,9 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_57] PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) + Select Operator [SEL_15] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_166] (rows=18262 width=1119) + Filter Operator [FIL_161] (rows=18262 width=1119) predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) TableScan [TS_13] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] @@ -232,9 +232,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_63] PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) + Select Operator [SEL_2] (rows=462000 width=1436) Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=462000 width=1436) + Filter Operator [FIL_158] (rows=462000 width=1436) predicate:(i_item_id is not null and i_item_sk is not null) TableScan [TS_0] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] @@ -246,11 +246,11 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_44] PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) + Group By Operator [GBY_6] (rows=231000 width=1436) Output:["_col0"],keys:i_item_id - Select Operator [SEL_42] (rows=231000 width=1436) + Select Operator [SEL_5] (rows=231000 width=1436) Output:["i_item_id"] - Filter Operator [FIL_164] (rows=231000 width=1436) + Filter Operator [FIL_159] (rows=231000 width=1436) predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null) TableScan [TS_3] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] @@ -276,11 +276,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_101] PartitionCols:_col1 - Select Operator [SEL_77] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_168] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 17 [SIMPLE_EDGE] SHUFFLE [RS_102] PartitionCols:_col0 @@ -289,13 +285,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_82] PartitionCols:_col0 - Group By Operator [GBY_81] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_80] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_169] (rows=231000 width=1436) - predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Group By Operator [GBY_6] <-Reducer 25 [SIMPLE_EDGE] SHUFFLE [RS_105] PartitionCols:_col3 @@ -306,11 +296,7 @@ Stage-0 <-Map 26 [SIMPLE_EDGE] SHUFFLE [RS_98] PartitionCols:_col0 - Select Operator [SEL_93] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_172] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) - Please refer to the previous TableScan [TS_16] + Please refer to the previous Select Operator [SEL_18] <-Reducer 24 [SIMPLE_EDGE] SHUFFLE [RS_97] PartitionCols:_col2 @@ -319,11 +305,7 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_95] PartitionCols:_col0 - Select Operator [SEL_90] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_171] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Map 28 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0 @@ -355,11 +337,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_26] PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_27] PartitionCols:_col0 @@ -368,13 +346,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_7] PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_159] (rows=231000 width=1436) - predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Group By Operator [GBY_6] <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col3 @@ -385,11 +357,7 @@ Stage-0 <-Map 26 [SIMPLE_EDGE] SHUFFLE [RS_23] PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_162] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) - Please refer to the previous TableScan [TS_16] + Please refer to the previous Select Operator [SEL_18] <-Reducer 19 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col2 @@ -398,11 +366,7 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_20] PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_161] (rows=18262 width=1119) - predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query57.q.out ql/src/test/results/clientpositive/perf/query57.q.out index 6c237bf..372e760 100644 --- ql/src/test/results/clientpositive/perf/query57.q.out +++ ql/src/test/results/clientpositive/perf/query57.q.out @@ -171,9 +171,9 @@ Stage-0 <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_52] PartitionCols:_col0 - Select Operator [SEL_44] (rows=462000 width=1436) + Select Operator [SEL_11] (rows=462000 width=1436) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_174] (rows=462000 width=1436) + Filter Operator [FIL_168] (rows=462000 width=1436) predicate:(i_item_sk is not null and i_category is not null and i_brand is not null) TableScan [TS_9] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"] @@ -185,9 +185,9 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_49] PartitionCols:_col0 - Select Operator [SEL_41] (rows=60 width=2045) + Select Operator [SEL_8] (rows=60 width=2045) Output:["_col0","_col1"] - Filter Operator [FIL_173] (rows=60 width=2045) + Filter Operator [FIL_167] (rows=60 width=2045) predicate:(cc_call_center_sk is not null and cc_name is not null) TableScan [TS_6] (rows=60 width=2045) default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_name"] @@ -199,18 +199,18 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_45] PartitionCols:_col0 - Select Operator [SEL_35] (rows=287989836 width=135) + Select Operator [SEL_2] (rows=287989836 width=135) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_171] (rows=287989836 width=135) + Filter Operator [FIL_165] (rows=287989836 width=135) predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_call_center_sk is not null) TableScan [TS_0] (rows=287989836 width=135) default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_call_center_sk","cs_item_sk","cs_sales_price"] <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_46] PartitionCols:_col0 - Select Operator [SEL_38] (rows=73048 width=1119) + Select Operator [SEL_5] (rows=73048 width=1119) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_172] (rows=73048 width=1119) + Filter Operator [FIL_166] (rows=73048 width=1119) predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] @@ -242,11 +242,7 @@ Stage-0 <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_89] PartitionCols:_col0 - Select Operator [SEL_81] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_179] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_category is not null and i_brand is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 16 [SIMPLE_EDGE] SHUFFLE [RS_88] PartitionCols:_col2 @@ -255,11 +251,7 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_86] PartitionCols:_col0 - Select Operator [SEL_78] (rows=60 width=2045) - Output:["_col0","_col1"] - Filter Operator [FIL_178] (rows=60 width=2045) - predicate:(cc_call_center_sk is not null and cc_name is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_85] PartitionCols:_col1 @@ -268,19 +260,11 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_82] PartitionCols:_col0 - Select Operator [SEL_72] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_176] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_call_center_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_83] PartitionCols:_col0 - Select Operator [SEL_75] (rows=73048 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_177] (rows=73048 width=1119) - predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_103] PartitionCols:_col0, _col2, _col1, (_col6 + 1) @@ -309,11 +293,7 @@ Stage-0 <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 - Select Operator [SEL_11] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_168] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_category is not null and i_brand is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col2 @@ -322,11 +302,7 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_8] (rows=60 width=2045) - Output:["_col0","_col1"] - Filter Operator [FIL_167] (rows=60 width=2045) - predicate:(cc_call_center_sk is not null and cc_name is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col1 @@ -335,17 +311,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_165] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_call_center_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_5] (rows=73048 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_166] (rows=73048 width=1119) - predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] diff --git ql/src/test/results/clientpositive/perf/query58.q.out ql/src/test/results/clientpositive/perf/query58.q.out index acdfc07..5b37f36 100644 --- ql/src/test/results/clientpositive/perf/query58.q.out +++ ql/src/test/results/clientpositive/perf/query58.q.out @@ -197,9 +197,9 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_85] PartitionCols:_col1 - Select Operator [SEL_58] (rows=73049 width=1119) + Select Operator [SEL_8] (rows=73049 width=1119) Output:["_col0","_col1"] - Filter Operator [FIL_254] (rows=73049 width=1119) + Filter Operator [FIL_248] (rows=73049 width=1119) predicate:(d_date is not null and d_date_sk is not null) TableScan [TS_6] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] @@ -245,10 +245,10 @@ Stage-0 Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 18 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_63] - Group By Operator [GBY_62] (rows=1 width=8) + Group By Operator [GBY_12] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_61] (rows=36524 width=1119) - Filter Operator [FIL_255] (rows=36524 width=1119) + Select Operator [SEL_11] (rows=36524 width=1119) + Filter Operator [FIL_249] (rows=36524 width=1119) predicate:(d_date = '1998-02-19') TableScan [TS_9] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date"] @@ -260,9 +260,9 @@ Stage-0 <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_90] PartitionCols:_col0 - Select Operator [SEL_55] (rows=462000 width=1436) + Select Operator [SEL_5] (rows=462000 width=1436) Output:["_col0","_col1"] - Filter Operator [FIL_253] (rows=462000 width=1436) + Filter Operator [FIL_247] (rows=462000 width=1436) predicate:(i_item_sk is not null and i_item_id is not null) TableScan [TS_3] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] @@ -295,11 +295,7 @@ Stage-0 <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_140] PartitionCols:_col0 - Select Operator [SEL_105] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_259] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_139] PartitionCols:_col1 @@ -317,11 +313,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_135] PartitionCols:_col1 - Select Operator [SEL_108] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_260] (rows=73049 width=1119) - predicate:(d_date is not null and d_date_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 30 [SIMPLE_EDGE] SHUFFLE [RS_136] PartitionCols:_col0 @@ -363,12 +355,7 @@ Stage-0 Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 18 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_113] - Group By Operator [GBY_112] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_111] (rows=36524 width=1119) - Filter Operator [FIL_261] (rows=36524 width=1119) - predicate:(d_date = '1998-02-19') - Please refer to the previous TableScan [TS_9] + Please refer to the previous Group By Operator [GBY_12] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_150] PartitionCols:_col0 @@ -389,11 +376,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_35] PartitionCols:_col1 - Select Operator [SEL_8] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_248] (rows=73049 width=1119) - predicate:(d_date is not null and d_date_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 22 [SIMPLE_EDGE] SHUFFLE [RS_36] PartitionCols:_col0 @@ -435,12 +418,7 @@ Stage-0 Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Map 18 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_13] - Group By Operator [GBY_12] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_11] (rows=36524 width=1119) - Filter Operator [FIL_249] (rows=36524 width=1119) - predicate:(d_date = '1998-02-19') - Please refer to the previous TableScan [TS_9] + Please refer to the previous Group By Operator [GBY_12] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_42] PartitionCols:_col0 @@ -449,11 +427,7 @@ Stage-0 <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_40] PartitionCols:_col0 - Select Operator [SEL_5] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_247] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_39] PartitionCols:_col1 diff --git ql/src/test/results/clientpositive/perf/query59.q.out ql/src/test/results/clientpositive/perf/query59.q.out index b570b96..baa9962 100644 --- ql/src/test/results/clientpositive/perf/query59.q.out +++ ql/src/test/results/clientpositive/perf/query59.q.out @@ -161,18 +161,18 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_34] PartitionCols:_col0 - Select Operator [SEL_30] (rows=575995635 width=88) + Select Operator [SEL_2] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_94] (rows=575995635 width=88) + Filter Operator [FIL_90] (rows=575995635 width=88) predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_sales_price"] <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_35] PartitionCols:_col0 - Select Operator [SEL_33] (rows=73049 width=1119) + Select Operator [SEL_5] (rows=73049 width=1119) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_95] (rows=73049 width=1119) + Filter Operator [FIL_91] (rows=73049 width=1119) predicate:(d_date_sk is not null and d_week_seq is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_week_seq","d_day_name"] @@ -222,17 +222,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_6] PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_90] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_7] PartitionCols:_col0 - Select Operator [SEL_5] (rows=73049 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_91] (rows=73049 width=1119) - predicate:(d_date_sk is not null and d_week_seq is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] diff --git ql/src/test/results/clientpositive/perf/query60.q.out ql/src/test/results/clientpositive/perf/query60.q.out index 4b7f8b3..7d63986 100644 --- ql/src/test/results/clientpositive/perf/query60.q.out +++ ql/src/test/results/clientpositive/perf/query60.q.out @@ -215,9 +215,9 @@ Stage-0 <-Map 26 [SIMPLE_EDGE] SHUFFLE [RS_60] PartitionCols:_col0 - Select Operator [SEL_55] (rows=20000000 width=1014) + Select Operator [SEL_18] (rows=20000000 width=1014) Output:["_col0"] - Filter Operator [FIL_167] (rows=20000000 width=1014) + Filter Operator [FIL_162] (rows=20000000 width=1014) predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) TableScan [TS_16] (rows=40000000 width=1014) default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] @@ -229,9 +229,9 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_57] PartitionCols:_col0 - Select Operator [SEL_52] (rows=18262 width=1119) + Select Operator [SEL_15] (rows=18262 width=1119) Output:["_col0"] - Filter Operator [FIL_166] (rows=18262 width=1119) + Filter Operator [FIL_161] (rows=18262 width=1119) predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) TableScan [TS_13] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] @@ -252,9 +252,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_63] PartitionCols:_col1 - Select Operator [SEL_39] (rows=462000 width=1436) + Select Operator [SEL_2] (rows=462000 width=1436) Output:["_col0","_col1"] - Filter Operator [FIL_163] (rows=462000 width=1436) + Filter Operator [FIL_158] (rows=462000 width=1436) predicate:(i_item_id is not null and i_item_sk is not null) TableScan [TS_0] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] @@ -266,11 +266,11 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_44] PartitionCols:_col0 - Group By Operator [GBY_43] (rows=231000 width=1436) + Group By Operator [GBY_6] (rows=231000 width=1436) Output:["_col0"],keys:i_item_id - Select Operator [SEL_42] (rows=231000 width=1436) + Select Operator [SEL_5] (rows=231000 width=1436) Output:["i_item_id"] - Filter Operator [FIL_164] (rows=231000 width=1436) + Filter Operator [FIL_159] (rows=231000 width=1436) predicate:((i_category) IN ('Children') and i_item_id is not null) TableScan [TS_3] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"] @@ -296,11 +296,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_101] PartitionCols:_col1 - Select Operator [SEL_77] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_168] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 17 [SIMPLE_EDGE] SHUFFLE [RS_102] PartitionCols:_col0 @@ -309,13 +305,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_82] PartitionCols:_col0 - Group By Operator [GBY_81] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_80] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_169] (rows=231000 width=1436) - predicate:((i_category) IN ('Children') and i_item_id is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Group By Operator [GBY_6] <-Reducer 25 [SIMPLE_EDGE] SHUFFLE [RS_105] PartitionCols:_col3 @@ -326,11 +316,7 @@ Stage-0 <-Map 26 [SIMPLE_EDGE] SHUFFLE [RS_98] PartitionCols:_col0 - Select Operator [SEL_93] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_172] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - Please refer to the previous TableScan [TS_16] + Please refer to the previous Select Operator [SEL_18] <-Reducer 24 [SIMPLE_EDGE] SHUFFLE [RS_97] PartitionCols:_col2 @@ -339,11 +325,7 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_95] PartitionCols:_col0 - Select Operator [SEL_90] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_171] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Map 28 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0 @@ -375,11 +357,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_26] PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_27] PartitionCols:_col0 @@ -388,13 +366,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_7] PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_159] (rows=231000 width=1436) - predicate:((i_category) IN ('Children') and i_item_id is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Group By Operator [GBY_6] <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col3 @@ -405,11 +377,7 @@ Stage-0 <-Map 26 [SIMPLE_EDGE] SHUFFLE [RS_23] PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_162] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - Please refer to the previous TableScan [TS_16] + Please refer to the previous Select Operator [SEL_18] <-Reducer 19 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col2 @@ -418,11 +386,7 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_20] PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_161] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null) - Please refer to the previous TableScan [TS_13] + Please refer to the previous Select Operator [SEL_15] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query61.q.out ql/src/test/results/clientpositive/perf/query61.q.out index 6838332..93d4d42 100644 --- ql/src/test/results/clientpositive/perf/query61.q.out +++ ql/src/test/results/clientpositive/perf/query61.q.out @@ -236,11 +236,7 @@ Stage-0 <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_70] PartitionCols:_col0 - Select Operator [SEL_62] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_144] (rows=852 width=1910) - predicate:((s_gmt_offset = -7) and s_store_sk is not null) - Please refer to the previous TableScan [TS_15] + Please refer to the previous Select Operator [SEL_17] <-Reducer 18 [SIMPLE_EDGE] SHUFFLE [RS_69] PartitionCols:_col3 @@ -249,11 +245,7 @@ Stage-0 <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_67] PartitionCols:_col0 - Select Operator [SEL_59] (rows=231000 width=1436) - Output:["_col0"] - Filter Operator [FIL_143] (rows=231000 width=1436) - predicate:((i_category = 'Electronics') and i_item_sk is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 17 [SIMPLE_EDGE] SHUFFLE [RS_66] PartitionCols:_col1 @@ -262,11 +254,7 @@ Stage-0 <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_64] PartitionCols:_col0 - Select Operator [SEL_56] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_142] (rows=18262 width=1119) - predicate:((d_year = 1999) and (d_moy = 11) and d_date_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_63] PartitionCols:_col0 @@ -284,17 +272,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_73] PartitionCols:_col1 - Select Operator [SEL_47] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_139] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_current_addr_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_74] PartitionCols:_col0 - Select Operator [SEL_50] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_140] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -7) and ca_address_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] diff --git ql/src/test/results/clientpositive/perf/query64.q.out ql/src/test/results/clientpositive/perf/query64.q.out index f24b14d..ddd0614 100644 --- ql/src/test/results/clientpositive/perf/query64.q.out +++ ql/src/test/results/clientpositive/perf/query64.q.out @@ -317,9 +317,9 @@ Stage-0 <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_247] PartitionCols:_col0 - Select Operator [SEL_227] (rows=1861800 width=385) + Select Operator [SEL_24] (rows=1861800 width=385) Output:["_col0","_col1"] - Filter Operator [FIL_574] (rows=1861800 width=385) + Filter Operator [FIL_543] (rows=1861800 width=385) predicate:cd_demo_sk is not null TableScan [TS_22] (rows=1861800 width=385) default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] @@ -336,11 +336,7 @@ Stage-0 <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_241] PartitionCols:_col0 - Select Operator [SEL_152] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_562] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - Please refer to the previous TableScan [TS_22] + Please refer to the previous Select Operator [SEL_24] <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_240] PartitionCols:_col1 @@ -349,9 +345,9 @@ Stage-0 <-Map 39 [SIMPLE_EDGE] SHUFFLE [RS_238] PartitionCols:_col0 - Select Operator [SEL_149] (rows=40000000 width=1014) + Select Operator [SEL_21] (rows=40000000 width=1014) Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_561] (rows=40000000 width=1014) + Filter Operator [FIL_542] (rows=40000000 width=1014) predicate:ca_address_sk is not null TableScan [TS_19] (rows=40000000 width=1014) default@customer_address,ad2,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] @@ -390,9 +386,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_228] PartitionCols:_col5 - Select Operator [SEL_130] (rows=80000000 width=860) + Select Operator [SEL_2] (rows=80000000 width=860) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_556] (rows=80000000 width=860) + Filter Operator [FIL_537] (rows=80000000 width=860) predicate:(c_customer_sk is not null and c_first_shipto_date_sk is not null and c_first_sales_date_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_current_addr_sk is not null) TableScan [TS_0] (rows=80000000 width=860) default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_shipto_date_sk","c_first_sales_date_sk"] @@ -404,18 +400,18 @@ Stage-0 <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_143] PartitionCols:_col1 - Select Operator [SEL_139] (rows=7200 width=107) + Select Operator [SEL_11] (rows=7200 width=107) Output:["_col0","_col1"] - Filter Operator [FIL_559] (rows=7200 width=107) + Filter Operator [FIL_540] (rows=7200 width=107) predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) TableScan [TS_9] (rows=7200 width=107) default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] <-Map 38 [SIMPLE_EDGE] SHUFFLE [RS_144] PartitionCols:_col0 - Select Operator [SEL_142] (rows=20 width=12) + Select Operator [SEL_14] (rows=20 width=12) Output:["_col0"] - Filter Operator [FIL_560] (rows=20 width=12) + Filter Operator [FIL_541] (rows=20 width=12) predicate:ib_income_band_sk is not null TableScan [TS_12] (rows=20 width=12) default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] @@ -429,11 +425,7 @@ Stage-0 <-Map 39 [SIMPLE_EDGE] SHUFFLE [RS_222] PartitionCols:_col0 - Select Operator [SEL_208] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_573] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - Please refer to the previous TableScan [TS_19] + Please refer to the previous Select Operator [SEL_21] <-Reducer 31 [SIMPLE_EDGE] SHUFFLE [RS_221] PartitionCols:_col13 @@ -442,9 +434,9 @@ Stage-0 <-Map 55 [SIMPLE_EDGE] SHUFFLE [RS_219] PartitionCols:_col0 - Select Operator [SEL_205] (rows=1704 width=1910) + Select Operator [SEL_77] (rows=1704 width=1910) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_572] (rows=1704 width=1910) + Filter Operator [FIL_553] (rows=1704 width=1910) predicate:(s_store_sk is not null and s_store_name is not null and s_zip is not null) TableScan [TS_75] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] @@ -468,9 +460,9 @@ Stage-0 <-Map 48 [SIMPLE_EDGE] SHUFFLE [RS_184] PartitionCols:_col0 - Select Operator [SEL_173] (rows=2300 width=1179) + Select Operator [SEL_45] (rows=2300 width=1179) Output:["_col0"] - Filter Operator [FIL_569] (rows=2300 width=1179) + Filter Operator [FIL_550] (rows=2300 width=1179) predicate:p_promo_sk is not null TableScan [TS_43] (rows=2300 width=1179) default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"] @@ -495,9 +487,9 @@ Stage-0 <-Map 47 [SIMPLE_EDGE] SHUFFLE [RS_178] PartitionCols:_col0, _col1 - Select Operator [SEL_167] (rows=57591150 width=77) + Select Operator [SEL_39] (rows=57591150 width=77) Output:["_col0","_col1"] - Filter Operator [FIL_567] (rows=57591150 width=77) + Filter Operator [FIL_548] (rows=57591150 width=77) predicate:(sr_item_sk is not null and sr_ticket_number is not null) TableScan [TS_37] (rows=57591150 width=77) default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] @@ -509,18 +501,18 @@ Stage-0 <-Map 41 [SIMPLE_EDGE] SHUFFLE [RS_174] PartitionCols:_col1 - Select Operator [SEL_161] (rows=575995635 width=88) + Select Operator [SEL_33] (rows=575995635 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_565] (rows=575995635 width=88) + Filter Operator [FIL_546] (rows=575995635 width=88) predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_customer_sk is not null and ss_cdemo_sk is not null and ss_promo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null) TableScan [TS_31] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] <-Map 46 [SIMPLE_EDGE] SHUFFLE [RS_175] PartitionCols:_col0 - Select Operator [SEL_164] (rows=2851 width=1436) + Select Operator [SEL_36] (rows=2851 width=1436) Output:["_col0","_col3"] - Filter Operator [FIL_566] (rows=2851 width=1436) + Filter Operator [FIL_547] (rows=2851 width=1436) predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null) TableScan [TS_34] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_color","i_product_name"] @@ -532,19 +524,11 @@ Stage-0 <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_209] PartitionCols:_col1 - Select Operator [SEL_155] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_563] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Map 38 [SIMPLE_EDGE] SHUFFLE [RS_210] PartitionCols:_col0 - Select Operator [SEL_158] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_564] (rows=20 width=12) - predicate:ib_income_band_sk is not null - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 53 [SIMPLE_EDGE] SHUFFLE [RS_216] PartitionCols:_col0 @@ -566,18 +550,18 @@ Stage-0 <-Map 49 [SIMPLE_EDGE] SHUFFLE [RS_193] PartitionCols:_col0, _col1 - Select Operator [SEL_189] (rows=287989836 width=135) + Select Operator [SEL_61] (rows=287989836 width=135) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_570] (rows=287989836 width=135) + Filter Operator [FIL_551] (rows=287989836 width=135) predicate:(cs_order_number is not null and cs_item_sk is not null) TableScan [TS_59] (rows=287989836 width=135) default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"] <-Map 54 [SIMPLE_EDGE] SHUFFLE [RS_194] PartitionCols:_col0, _col1 - Select Operator [SEL_192] (rows=28798881 width=106) + Select Operator [SEL_64] (rows=28798881 width=106) Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_571] (rows=28798881 width=106) + Filter Operator [FIL_552] (rows=28798881 width=106) predicate:(cr_order_number is not null and cr_item_sk is not null) TableScan [TS_62] (rows=28798881 width=106) default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"] @@ -602,11 +586,7 @@ Stage-0 <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_119] PartitionCols:_col0 - Select Operator [SEL_99] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_555] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - Please refer to the previous TableScan [TS_22] + Please refer to the previous Select Operator [SEL_24] <-Reducer 7 [SIMPLE_EDGE] SHUFFLE [RS_118] PartitionCols:_col39 @@ -622,11 +602,7 @@ Stage-0 <-Map 39 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0 - Select Operator [SEL_80] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_554] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - Please refer to the previous TableScan [TS_19] + Please refer to the previous Select Operator [SEL_21] <-Reducer 25 [SIMPLE_EDGE] SHUFFLE [RS_93] PartitionCols:_col13 @@ -635,11 +611,7 @@ Stage-0 <-Map 55 [SIMPLE_EDGE] SHUFFLE [RS_91] PartitionCols:_col0 - Select Operator [SEL_77] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_553] (rows=1704 width=1910) - predicate:(s_store_sk is not null and s_store_name is not null and s_zip is not null) - Please refer to the previous TableScan [TS_75] + Please refer to the previous Select Operator [SEL_77] <-Reducer 24 [SIMPLE_EDGE] SHUFFLE [RS_90] PartitionCols:_col14 @@ -660,11 +632,7 @@ Stage-0 <-Map 48 [SIMPLE_EDGE] SHUFFLE [RS_56] PartitionCols:_col0 - Select Operator [SEL_45] (rows=2300 width=1179) - Output:["_col0"] - Filter Operator [FIL_550] (rows=2300 width=1179) - predicate:p_promo_sk is not null - Please refer to the previous TableScan [TS_43] + Please refer to the previous Select Operator [SEL_45] <-Reducer 21 [SIMPLE_EDGE] SHUFFLE [RS_55] PartitionCols:_col7 @@ -686,11 +654,7 @@ Stage-0 <-Map 47 [SIMPLE_EDGE] SHUFFLE [RS_50] PartitionCols:_col0, _col1 - Select Operator [SEL_39] (rows=57591150 width=77) - Output:["_col0","_col1"] - Filter Operator [FIL_548] (rows=57591150 width=77) - predicate:(sr_item_sk is not null and sr_ticket_number is not null) - Please refer to the previous TableScan [TS_37] + Please refer to the previous Select Operator [SEL_39] <-Reducer 42 [SIMPLE_EDGE] SHUFFLE [RS_49] PartitionCols:_col1, _col8 @@ -699,19 +663,11 @@ Stage-0 <-Map 41 [SIMPLE_EDGE] SHUFFLE [RS_46] PartitionCols:_col1 - Select Operator [SEL_33] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"] - Filter Operator [FIL_546] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_customer_sk is not null and ss_cdemo_sk is not null and ss_promo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null) - Please refer to the previous TableScan [TS_31] + Please refer to the previous Select Operator [SEL_33] <-Map 46 [SIMPLE_EDGE] SHUFFLE [RS_47] PartitionCols:_col0 - Select Operator [SEL_36] (rows=2851 width=1436) - Output:["_col0","_col3"] - Filter Operator [FIL_547] (rows=2851 width=1436) - predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null) - Please refer to the previous TableScan [TS_34] + Please refer to the previous Select Operator [SEL_36] <-Reducer 35 [SIMPLE_EDGE] SHUFFLE [RS_84] PartitionCols:_col0 @@ -720,19 +676,11 @@ Stage-0 <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_81] PartitionCols:_col1 - Select Operator [SEL_27] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_544] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Map 38 [SIMPLE_EDGE] SHUFFLE [RS_82] PartitionCols:_col0 - Select Operator [SEL_30] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_545] (rows=20 width=12) - predicate:ib_income_band_sk is not null - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 51 [SIMPLE_EDGE] SHUFFLE [RS_88] PartitionCols:_col0 @@ -754,19 +702,11 @@ Stage-0 <-Map 49 [SIMPLE_EDGE] SHUFFLE [RS_65] PartitionCols:_col0, _col1 - Select Operator [SEL_61] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_551] (rows=287989836 width=135) - predicate:(cs_order_number is not null and cs_item_sk is not null) - Please refer to the previous TableScan [TS_59] + Please refer to the previous Select Operator [SEL_61] <-Map 54 [SIMPLE_EDGE] SHUFFLE [RS_66] PartitionCols:_col0, _col1 - Select Operator [SEL_64] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_552] (rows=28798881 width=106) - predicate:(cr_order_number is not null and cr_item_sk is not null) - Please refer to the previous TableScan [TS_62] + Please refer to the previous Select Operator [SEL_64] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_115] PartitionCols:_col0 @@ -775,11 +715,7 @@ Stage-0 <-Map 40 [SIMPLE_EDGE] SHUFFLE [RS_113] PartitionCols:_col0 - Select Operator [SEL_24] (rows=1861800 width=385) - Output:["_col0","_col1"] - Filter Operator [FIL_543] (rows=1861800 width=385) - predicate:cd_demo_sk is not null - Please refer to the previous TableScan [TS_22] + Please refer to the previous Select Operator [SEL_24] <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_112] PartitionCols:_col1 @@ -788,11 +724,7 @@ Stage-0 <-Map 39 [SIMPLE_EDGE] SHUFFLE [RS_110] PartitionCols:_col0 - Select Operator [SEL_21] (rows=40000000 width=1014) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_542] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - Please refer to the previous TableScan [TS_19] + Please refer to the previous Select Operator [SEL_21] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_109] PartitionCols:_col3 @@ -806,9 +738,9 @@ Stage-0 <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_104] PartitionCols:_col0 - Select Operator [SEL_8] (rows=73049 width=1119) + Select Operator [SEL_5] (rows=73049 width=1119) Output:["_col0","_col1"] - Filter Operator [FIL_539] (rows=73049 width=1119) + Filter Operator [FIL_538] (rows=73049 width=1119) predicate:d_date_sk is not null Please refer to the previous TableScan [TS_3] <-Reducer 2 [SIMPLE_EDGE] @@ -819,19 +751,11 @@ Stage-0 <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_101] PartitionCols:_col0 - Select Operator [SEL_5] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_538] (rows=73049 width=1119) - predicate:d_date_sk is not null - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_100] PartitionCols:_col5 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_537] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_first_shipto_date_sk is not null and c_first_sales_date_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_current_addr_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 34 [SIMPLE_EDGE] SHUFFLE [RS_107] PartitionCols:_col0 @@ -840,17 +764,9 @@ Stage-0 <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col1 - Select Operator [SEL_11] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_540] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Map 38 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_14] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_541] (rows=20 width=12) - predicate:ib_income_band_sk is not null - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] diff --git ql/src/test/results/clientpositive/perf/query65.q.out ql/src/test/results/clientpositive/perf/query65.q.out index b2035c2..1b32ed6 100644 --- ql/src/test/results/clientpositive/perf/query65.q.out +++ ql/src/test/results/clientpositive/perf/query65.q.out @@ -165,9 +165,5 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_21] PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_75] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] diff --git ql/src/test/results/clientpositive/perf/query66.q.out ql/src/test/results/clientpositive/perf/query66.q.out index 2c74815..9a37cf3 100644 --- ql/src/test/results/clientpositive/perf/query66.q.out +++ ql/src/test/results/clientpositive/perf/query66.q.out @@ -494,9 +494,9 @@ Stage-0 <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_58] PartitionCols:_col0 - Select Operator [SEL_47] (rows=27 width=1029) + Select Operator [SEL_14] (rows=27 width=1029) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_115] (rows=27 width=1029) + Filter Operator [FIL_110] (rows=27 width=1029) predicate:w_warehouse_sk is not null TableScan [TS_12] (rows=27 width=1029) default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"] @@ -508,9 +508,9 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_55] PartitionCols:_col0 - Select Operator [SEL_44] (rows=1 width=0) + Select Operator [SEL_11] (rows=1 width=0) Output:["_col0"] - Filter Operator [FIL_114] (rows=1 width=0) + Filter Operator [FIL_109] (rows=1 width=0) predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) TableScan [TS_9] (rows=1 width=0) default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"] @@ -522,9 +522,9 @@ Stage-0 <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_52] PartitionCols:_col0 - Select Operator [SEL_41] (rows=36524 width=1119) + Select Operator [SEL_8] (rows=36524 width=1119) Output:["_col0","_col2"] - Filter Operator [FIL_113] (rows=36524 width=1119) + Filter Operator [FIL_108] (rows=36524 width=1119) predicate:((d_year = 2002) and d_date_sk is not null) TableScan [TS_6] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] @@ -536,9 +536,9 @@ Stage-0 <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_49] PartitionCols:_col0 - Select Operator [SEL_38] (rows=9600 width=471) + Select Operator [SEL_5] (rows=9600 width=471) Output:["_col0"] - Filter Operator [FIL_112] (rows=9600 width=471) + Filter Operator [FIL_107] (rows=9600 width=471) predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) TableScan [TS_3] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"] @@ -572,11 +572,7 @@ Stage-0 <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_25] PartitionCols:_col0 - Select Operator [SEL_14] (rows=27 width=1029) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_110] (rows=27 width=1029) - predicate:w_warehouse_sk is not null - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_24] PartitionCols:_col3 @@ -585,11 +581,7 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col0 - Select Operator [SEL_11] (rows=1 width=0) - Output:["_col0"] - Filter Operator [FIL_109] (rows=1 width=0) - predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_21] PartitionCols:_col2 @@ -598,11 +590,7 @@ Stage-0 <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 - Select Operator [SEL_8] (rows=36524 width=1119) - Output:["_col0","_col2"] - Filter Operator [FIL_108] (rows=36524 width=1119) - predicate:((d_year = 2002) and d_date_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col0 @@ -611,11 +599,7 @@ Stage-0 <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_5] (rows=9600 width=471) - Output:["_col0"] - Filter Operator [FIL_107] (rows=9600 width=471) - predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col1 diff --git ql/src/test/results/clientpositive/perf/query68.q.out ql/src/test/results/clientpositive/perf/query68.q.out index bd9b5ec..5f763d5 100644 --- ql/src/test/results/clientpositive/perf/query68.q.out +++ ql/src/test/results/clientpositive/perf/query68.q.out @@ -150,11 +150,7 @@ Stage-0 <-Map 5 [SIMPLE_EDGE] SHUFFLE [RS_31] PartitionCols:_col0 - Select Operator [SEL_20] (rows=40000000 width=1014) - Output:["_col0","_col1"] - Filter Operator [FIL_80] (rows=40000000 width=1014) - predicate:ca_address_sk is not null - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col3 diff --git ql/src/test/results/clientpositive/perf/query69.q.out ql/src/test/results/clientpositive/perf/query69.q.out index a55c368..2769bc5 100644 --- ql/src/test/results/clientpositive/perf/query69.q.out +++ ql/src/test/results/clientpositive/perf/query69.q.out @@ -150,9 +150,9 @@ Stage-0 <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_57] PartitionCols:_col0 - Select Operator [SEL_55] (rows=4058 width=1119) + Select Operator [SEL_14] (rows=4058 width=1119) Output:["_col0"] - Filter Operator [FIL_107] (rows=4058 width=1119) + Filter Operator [FIL_103] (rows=4058 width=1119) predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) TableScan [TS_12] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] @@ -193,11 +193,7 @@ Stage-0 <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_14] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_103] (rows=4058 width=1119) - predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col0 @@ -222,11 +218,7 @@ Stage-0 <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col0 - Select Operator [SEL_28] (rows=4058 width=1119) - Output:["_col0"] - Filter Operator [FIL_105] (rows=4058 width=1119) - predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_29] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query70.q.out ql/src/test/results/clientpositive/perf/query70.q.out index ee1fe86..9f9a8e1 100644 --- ql/src/test/results/clientpositive/perf/query70.q.out +++ ql/src/test/results/clientpositive/perf/query70.q.out @@ -160,18 +160,18 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col0 - Select Operator [SEL_11] (rows=575995635 width=88) + Select Operator [SEL_2] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_82] (rows=575995635 width=88) - predicate:(ss_store_sk is not null and ss_sold_date_sk is not null) + Filter Operator [FIL_78] (rows=575995635 width=88) + predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 - Select Operator [SEL_14] (rows=8116 width=1119) + Select Operator [SEL_5] (rows=8116 width=1119) Output:["_col0"] - Filter Operator [FIL_83] (rows=8116 width=1119) + Filter Operator [FIL_79] (rows=8116 width=1119) predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] @@ -197,17 +197,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_37] PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_78] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_79] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] diff --git ql/src/test/results/clientpositive/perf/query74.q.out ql/src/test/results/clientpositive/perf/query74.q.out index 50d97e7..9dddd6a 100644 --- ql/src/test/results/clientpositive/perf/query74.q.out +++ ql/src/test/results/clientpositive/perf/query74.q.out @@ -173,9 +173,9 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_33] PartitionCols:_col0 - Select Operator [SEL_28] (rows=80000000 width=860) + Select Operator [SEL_8] (rows=80000000 width=860) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_154] (rows=80000000 width=860) + Filter Operator [FIL_151] (rows=80000000 width=860) predicate:(c_customer_sk is not null and c_customer_id is not null) TableScan [TS_6] (rows=80000000 width=860) default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_first_name","c_last_name"] @@ -219,11 +219,7 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_75] PartitionCols:_col0 - Select Operator [SEL_70] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_160] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_customer_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 14 [SIMPLE_EDGE] SHUFFLE [RS_74] PartitionCols:_col1 @@ -240,11 +236,7 @@ Stage-0 <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_71] PartitionCols:_col0 - Select Operator [SEL_64] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_158] (rows=575995635 width=88) - predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) - Please refer to the previous TableScan [TS_20] + Please refer to the previous Select Operator [SEL_22] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_82] PartitionCols:_col0 @@ -262,11 +254,7 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_151] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_customer_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col1 @@ -308,11 +296,7 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_54] PartitionCols:_col0 - Select Operator [SEL_49] (rows=80000000 width=860) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_157] (rows=80000000 width=860) - predicate:(c_customer_sk is not null and c_customer_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 7 [SIMPLE_EDGE] SHUFFLE [RS_53] PartitionCols:_col1 @@ -329,9 +313,5 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_50] PartitionCols:_col0 - Select Operator [SEL_43] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_155] (rows=144002668 width=135) - predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] diff --git ql/src/test/results/clientpositive/perf/query75.q.out ql/src/test/results/clientpositive/perf/query75.q.out index d399567..c3dfff5 100644 --- ql/src/test/results/clientpositive/perf/query75.q.out +++ ql/src/test/results/clientpositive/perf/query75.q.out @@ -212,9 +212,9 @@ Stage-0 <-Map 32 [SIMPLE_EDGE] SHUFFLE [RS_104] PartitionCols:_col0, _col1 - Select Operator [SEL_96] (rows=28798881 width=106) + Select Operator [SEL_11] (rows=28798881 width=106) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_255] (rows=28798881 width=106) + Filter Operator [FIL_243] (rows=28798881 width=106) predicate:cr_item_sk is not null TableScan [TS_9] (rows=28798881 width=106) default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"] @@ -226,9 +226,9 @@ Stage-0 <-Map 31 [SIMPLE_EDGE] SHUFFLE [RS_101] PartitionCols:_col0 - Select Operator [SEL_93] (rows=231000 width=1436) + Select Operator [SEL_8] (rows=231000 width=1436) Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_254] (rows=231000 width=1436) + Filter Operator [FIL_242] (rows=231000 width=1436) predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) TableScan [TS_6] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"] @@ -249,9 +249,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_97] PartitionCols:_col0 - Select Operator [SEL_87] (rows=287989836 width=135) + Select Operator [SEL_2] (rows=287989836 width=135) Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_252] (rows=287989836 width=135) + Filter Operator [FIL_240] (rows=287989836 width=135) predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) TableScan [TS_0] (rows=287989836 width=135) default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"] @@ -267,9 +267,9 @@ Stage-0 <-Map 34 [SIMPLE_EDGE] SHUFFLE [RS_126] PartitionCols:_col0, _col1 - Select Operator [SEL_118] (rows=57591150 width=77) + Select Operator [SEL_33] (rows=57591150 width=77) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_259] (rows=57591150 width=77) + Filter Operator [FIL_247] (rows=57591150 width=77) predicate:sr_item_sk is not null TableScan [TS_31] (rows=57591150 width=77) default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"] @@ -281,11 +281,7 @@ Stage-0 <-Map 31 [SIMPLE_EDGE] SHUFFLE [RS_123] PartitionCols:_col0 - Select Operator [SEL_115] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_258] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 25 [SIMPLE_EDGE] SHUFFLE [RS_122] PartitionCols:_col1 @@ -302,9 +298,9 @@ Stage-0 <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_119] PartitionCols:_col0 - Select Operator [SEL_109] (rows=575995635 width=88) + Select Operator [SEL_24] (rows=575995635 width=88) Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_256] (rows=575995635 width=88) + Filter Operator [FIL_244] (rows=575995635 width=88) predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) TableScan [TS_22] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"] @@ -320,9 +316,9 @@ Stage-0 <-Map 36 [SIMPLE_EDGE] SHUFFLE [RS_155] PartitionCols:_col0, _col1 - Select Operator [SEL_147] (rows=14398467 width=92) + Select Operator [SEL_62] (rows=14398467 width=92) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_263] (rows=14398467 width=92) + Filter Operator [FIL_251] (rows=14398467 width=92) predicate:wr_item_sk is not null TableScan [TS_60] (rows=14398467 width=92) default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"] @@ -334,11 +330,7 @@ Stage-0 <-Map 31 [SIMPLE_EDGE] SHUFFLE [RS_152] PartitionCols:_col0 - Select Operator [SEL_144] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_262] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 28 [SIMPLE_EDGE] SHUFFLE [RS_151] PartitionCols:_col1 @@ -355,9 +347,9 @@ Stage-0 <-Map 35 [SIMPLE_EDGE] SHUFFLE [RS_148] PartitionCols:_col0 - Select Operator [SEL_138] (rows=144002668 width=135) + Select Operator [SEL_53] (rows=144002668 width=135) Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_260] (rows=144002668 width=135) + Filter Operator [FIL_248] (rows=144002668 width=135) predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) TableScan [TS_51] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"] @@ -381,11 +373,7 @@ Stage-0 <-Map 36 [SIMPLE_EDGE] SHUFFLE [RS_70] PartitionCols:_col0, _col1 - Select Operator [SEL_62] (rows=14398467 width=92) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_251] (rows=14398467 width=92) - predicate:wr_item_sk is not null - Please refer to the previous TableScan [TS_60] + Please refer to the previous Select Operator [SEL_62] <-Reducer 23 [SIMPLE_EDGE] SHUFFLE [RS_69] PartitionCols:_col1, _col2 @@ -394,11 +382,7 @@ Stage-0 <-Map 31 [SIMPLE_EDGE] SHUFFLE [RS_67] PartitionCols:_col0 - Select Operator [SEL_59] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_250] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 22 [SIMPLE_EDGE] SHUFFLE [RS_66] PartitionCols:_col1 @@ -407,19 +391,15 @@ Stage-0 <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_64] PartitionCols:_col0 - Select Operator [SEL_56] (rows=36524 width=1119) + Select Operator [SEL_5] (rows=36524 width=1119) Output:["_col0"] - Filter Operator [FIL_249] (rows=36524 width=1119) + Filter Operator [FIL_241] (rows=36524 width=1119) predicate:((d_year = 2001) and d_date_sk is not null) Please refer to the previous TableScan [TS_3] <-Map 35 [SIMPLE_EDGE] SHUFFLE [RS_63] PartitionCols:_col0 - Select Operator [SEL_53] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_248] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - Please refer to the previous TableScan [TS_51] + Please refer to the previous Select Operator [SEL_53] <-Reducer 6 [CONTAINS] Reduce Output Operator [RS_77] PartitionCols:_col0, _col1, _col2, _col3 @@ -440,11 +420,7 @@ Stage-0 <-Map 34 [SIMPLE_EDGE] SHUFFLE [RS_41] PartitionCols:_col0, _col1 - Select Operator [SEL_33] (rows=57591150 width=77) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_247] (rows=57591150 width=77) - predicate:sr_item_sk is not null - Please refer to the previous TableScan [TS_31] + Please refer to the previous Select Operator [SEL_33] <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_40] PartitionCols:_col1, _col2 @@ -453,11 +429,7 @@ Stage-0 <-Map 31 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:_col0 - Select Operator [SEL_30] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_246] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 19 [SIMPLE_EDGE] SHUFFLE [RS_37] PartitionCols:_col1 @@ -466,19 +438,11 @@ Stage-0 <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_35] PartitionCols:_col0 - Select Operator [SEL_27] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_245] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_34] PartitionCols:_col0 - Select Operator [SEL_24] (rows=575995635 width=88) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_244] (rows=575995635 width=88) - predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) - Please refer to the previous TableScan [TS_22] + Please refer to the previous Select Operator [SEL_24] <-Reducer 4 [CONTAINS] Reduce Output Operator [RS_48] PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 @@ -491,11 +455,7 @@ Stage-0 <-Map 32 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0, _col1 - Select Operator [SEL_11] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_243] (rows=28798881 width=106) - predicate:cr_item_sk is not null - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col1, _col2 @@ -504,11 +464,7 @@ Stage-0 <-Map 31 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_8] (rows=231000 width=1436) - Output:["_col0","_col1","_col2","_col3","_col5"] - Filter Operator [FIL_242] (rows=231000 width=1436) - predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col1 @@ -517,17 +473,9 @@ Stage-0 <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_5] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_241] (rows=36524 width=1119) - predicate:((d_year = 2001) and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col0 - Select Operator [SEL_2] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_240] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] diff --git ql/src/test/results/clientpositive/perf/query76.q.out ql/src/test/results/clientpositive/perf/query76.q.out index e7fa6df..de59732 100644 --- ql/src/test/results/clientpositive/perf/query76.q.out +++ ql/src/test/results/clientpositive/perf/query76.q.out @@ -97,9 +97,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_43] PartitionCols:_col0 - Select Operator [SEL_38] (rows=462000 width=1436) + Select Operator [SEL_2] (rows=462000 width=1436) Output:["_col0","_col1"] - Filter Operator [FIL_81] (rows=462000 width=1436) + Filter Operator [FIL_74] (rows=462000 width=1436) predicate:i_item_sk is not null TableScan [TS_0] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"] @@ -138,11 +138,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_9] PartitionCols:_col0 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_74] (rows=462000 width=1436) - predicate:i_item_sk is not null - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_10] PartitionCols:_col1 @@ -178,11 +174,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_26] PartitionCols:_col0 - Select Operator [SEL_21] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_78] (rows=462000 width=1436) - predicate:i_item_sk is not null - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_25] PartitionCols:_col1 diff --git ql/src/test/results/clientpositive/perf/query77.q.out ql/src/test/results/clientpositive/perf/query77.q.out index 04aa84b..db42b18 100644 --- ql/src/test/results/clientpositive/perf/query77.q.out +++ ql/src/test/results/clientpositive/perf/query77.q.out @@ -276,9 +276,9 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_51] PartitionCols:_col0 - Select Operator [SEL_49] (rows=8116 width=1119) + Select Operator [SEL_5] (rows=8116 width=1119) Output:["_col0"] - Filter Operator [FIL_168] (rows=8116 width=1119) + Filter Operator [FIL_162] (rows=8116 width=1119) predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] @@ -304,11 +304,7 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_65] PartitionCols:_col0 - Select Operator [SEL_63] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_170] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 28 [SIMPLE_EDGE] SHUFFLE [RS_64] PartitionCols:_col0 @@ -356,11 +352,7 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_87] PartitionCols:_col0 - Select Operator [SEL_82] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_172] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 29 [SIMPLE_EDGE] SHUFFLE [RS_86] PartitionCols:_col0 @@ -385,11 +377,7 @@ Stage-0 <-Map 30 [SIMPLE_EDGE] SHUFFLE [RS_110] PartitionCols:_col0 - Select Operator [SEL_105] (rows=4602 width=585) - Output:["_col0"] - Filter Operator [FIL_176] (rows=4602 width=585) - predicate:wp_web_page_sk is not null - Please refer to the previous TableScan [TS_83] + Please refer to the previous Select Operator [SEL_85] <-Reducer 22 [SIMPLE_EDGE] SHUFFLE [RS_109] PartitionCols:_col1 @@ -398,11 +386,7 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_107] PartitionCols:_col0 - Select Operator [SEL_102] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_175] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 31 [SIMPLE_EDGE] SHUFFLE [RS_106] PartitionCols:_col0 @@ -436,9 +420,9 @@ Stage-0 <-Map 25 [SIMPLE_EDGE] SHUFFLE [RS_33] PartitionCols:_col0 - Select Operator [SEL_28] (rows=1704 width=1910) + Select Operator [SEL_8] (rows=1704 width=1910) Output:["_col0"] - Filter Operator [FIL_166] (rows=1704 width=1910) + Filter Operator [FIL_163] (rows=1704 width=1910) predicate:s_store_sk is not null TableScan [TS_6] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk"] @@ -450,11 +434,7 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col0 - Select Operator [SEL_25] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_165] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 26 [SIMPLE_EDGE] SHUFFLE [RS_29] PartitionCols:_col0 @@ -479,11 +459,7 @@ Stage-0 <-Map 25 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_8] (rows=1704 width=1910) - Output:["_col0"] - Filter Operator [FIL_163] (rows=1704 width=1910) - predicate:s_store_sk is not null - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col1 @@ -492,11 +468,7 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_10] PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_162] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_9] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query78.q.out ql/src/test/results/clientpositive/perf/query78.q.out index b49103f..2fccac2 100644 --- ql/src/test/results/clientpositive/perf/query78.q.out +++ ql/src/test/results/clientpositive/perf/query78.q.out @@ -165,9 +165,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_60] PartitionCols:_col0 - Select Operator [SEL_49] (rows=36524 width=1119) + Select Operator [SEL_2] (rows=36524 width=1119) Output:["_col0"] - Filter Operator [FIL_103] (rows=36524 width=1119) + Filter Operator [FIL_97] (rows=36524 width=1119) predicate:((d_year = 2000) and d_date_sk is not null) TableScan [TS_0] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] @@ -220,11 +220,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_2] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_97] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_14] PartitionCols:_col0 @@ -267,11 +263,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_34] PartitionCols:_col0 - Select Operator [SEL_23] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_100] (rows=36524 width=1119) - predicate:((d_year = 2000) and d_date_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_35] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query80.q.out ql/src/test/results/clientpositive/perf/query80.q.out index e37b50d..b717bc5 100644 --- ql/src/test/results/clientpositive/perf/query80.q.out +++ ql/src/test/results/clientpositive/perf/query80.q.out @@ -264,9 +264,9 @@ Stage-0 <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_67] PartitionCols:_col0 - Select Operator [SEL_53] (rows=1150 width=1179) + Select Operator [SEL_14] (rows=1150 width=1179) Output:["_col0"] - Filter Operator [FIL_195] (rows=1150 width=1179) + Filter Operator [FIL_189] (rows=1150 width=1179) predicate:((p_channel_tv = 'N') and p_promo_sk is not null) TableScan [TS_12] (rows=2300 width=1179) default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_tv"] @@ -278,9 +278,9 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_64] PartitionCols:_col0 - Select Operator [SEL_50] (rows=154000 width=1436) + Select Operator [SEL_11] (rows=154000 width=1436) Output:["_col0"] - Filter Operator [FIL_194] (rows=154000 width=1436) + Filter Operator [FIL_188] (rows=154000 width=1436) predicate:((i_current_price > 50) and i_item_sk is not null) TableScan [TS_9] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price"] @@ -292,9 +292,9 @@ Stage-0 <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_61] PartitionCols:_col0 - Select Operator [SEL_47] (rows=8116 width=1119) + Select Operator [SEL_8] (rows=8116 width=1119) Output:["_col0"] - Filter Operator [FIL_193] (rows=8116 width=1119) + Filter Operator [FIL_187] (rows=8116 width=1119) predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) TableScan [TS_6] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] @@ -356,11 +356,7 @@ Stage-0 <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_107] PartitionCols:_col0 - Select Operator [SEL_93] (rows=1150 width=1179) - Output:["_col0"] - Filter Operator [FIL_201] (rows=1150 width=1179) - predicate:((p_channel_tv = 'N') and p_promo_sk is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 19 [SIMPLE_EDGE] SHUFFLE [RS_106] PartitionCols:_col3 @@ -369,11 +365,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_104] PartitionCols:_col0 - Select Operator [SEL_90] (rows=154000 width=1436) - Output:["_col0"] - Filter Operator [FIL_200] (rows=154000 width=1436) - predicate:((i_current_price > 50) and i_item_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 18 [SIMPLE_EDGE] SHUFFLE [RS_103] PartitionCols:_col1 @@ -382,11 +374,7 @@ Stage-0 <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_101] PartitionCols:_col0 - Select Operator [SEL_87] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_199] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 31 [SIMPLE_EDGE] SHUFFLE [RS_100] PartitionCols:_col0 @@ -445,11 +433,7 @@ Stage-0 <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_28] PartitionCols:_col0 - Select Operator [SEL_14] (rows=1150 width=1179) - Output:["_col0"] - Filter Operator [FIL_189] (rows=1150 width=1179) - predicate:((p_channel_tv = 'N') and p_promo_sk is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_27] PartitionCols:_col3 @@ -458,11 +442,7 @@ Stage-0 <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_25] PartitionCols:_col0 - Select Operator [SEL_11] (rows=154000 width=1436) - Output:["_col0"] - Filter Operator [FIL_188] (rows=154000 width=1436) - predicate:((i_current_price > 50) and i_item_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_24] PartitionCols:_col1 @@ -471,11 +451,7 @@ Stage-0 <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col0 - Select Operator [SEL_8] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_187] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_21] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query81.q.out ql/src/test/results/clientpositive/perf/query81.q.out index abeb577..c7571fe 100644 --- ql/src/test/results/clientpositive/perf/query81.q.out +++ ql/src/test/results/clientpositive/perf/query81.q.out @@ -134,9 +134,9 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_36] PartitionCols:_col0 - Select Operator [SEL_31] (rows=36524 width=1119) + Select Operator [SEL_11] (rows=36524 width=1119) Output:["_col0"] - Filter Operator [FIL_98] (rows=36524 width=1119) + Filter Operator [FIL_95] (rows=36524 width=1119) predicate:((d_year = 1998) and d_date_sk is not null) TableScan [TS_9] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] @@ -179,11 +179,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_11] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_95] (rows=36524 width=1119) - predicate:((d_year = 1998) and d_date_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Map 6 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query83.q.out ql/src/test/results/clientpositive/perf/query83.q.out index 396c423..a82dbb8 100644 --- ql/src/test/results/clientpositive/perf/query83.q.out +++ ql/src/test/results/clientpositive/perf/query83.q.out @@ -193,9 +193,9 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_69] PartitionCols:_col1 - Select Operator [SEL_50] (rows=73049 width=1119) + Select Operator [SEL_8] (rows=73049 width=1119) Output:["_col0","_col1"] - Filter Operator [FIL_208] (rows=73049 width=1119) + Filter Operator [FIL_203] (rows=73049 width=1119) predicate:(d_date is not null and d_date_sk is not null) TableScan [TS_6] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] @@ -243,9 +243,9 @@ Stage-0 <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_74] PartitionCols:_col0 - Select Operator [SEL_47] (rows=462000 width=1436) + Select Operator [SEL_5] (rows=462000 width=1436) Output:["_col0","_col1"] - Filter Operator [FIL_207] (rows=462000 width=1436) + Filter Operator [FIL_202] (rows=462000 width=1436) predicate:(i_item_sk is not null and i_item_id is not null) TableScan [TS_3] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] @@ -278,11 +278,7 @@ Stage-0 <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_116] PartitionCols:_col0 - Select Operator [SEL_89] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_212] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 29 [SIMPLE_EDGE] SHUFFLE [RS_115] PartitionCols:_col1 @@ -300,11 +296,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_111] PartitionCols:_col1 - Select Operator [SEL_92] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_213] (rows=73049 width=1119) - predicate:(d_date is not null and d_date_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 26 [SIMPLE_EDGE] SHUFFLE [RS_112] PartitionCols:_col0 @@ -360,11 +352,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_27] PartitionCols:_col1 - Select Operator [SEL_8] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_203] (rows=73049 width=1119) - predicate:(d_date is not null and d_date_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_28] PartitionCols:_col0 @@ -408,11 +396,7 @@ Stage-0 <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_32] PartitionCols:_col0 - Select Operator [SEL_5] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_202] (rows=462000 width=1436) - predicate:(i_item_sk is not null and i_item_id is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_31] PartitionCols:_col1 diff --git ql/src/test/results/clientpositive/perf/query85.q.out ql/src/test/results/clientpositive/perf/query85.q.out index 86b961b..07e2e48 100644 --- ql/src/test/results/clientpositive/perf/query85.q.out +++ ql/src/test/results/clientpositive/perf/query85.q.out @@ -203,9 +203,9 @@ Stage-0 <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_45] PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_23] (rows=1861800 width=385) + Select Operator [SEL_20] (rows=1861800 width=385) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_99] (rows=1861800 width=385) + Filter Operator [FIL_98] (rows=1861800 width=385) predicate:(((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_marital_status is not null and cd_education_status is not null) TableScan [TS_18] (rows=1861800 width=385) default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"] @@ -219,11 +219,7 @@ Stage-0 <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_41] PartitionCols:_col0 - Select Operator [SEL_20] (rows=1861800 width=385) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_98] (rows=1861800 width=385) - predicate:(((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_marital_status is not null and cd_education_status is not null) - Please refer to the previous TableScan [TS_18] + Please refer to the previous Select Operator [SEL_20] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_40] PartitionCols:_col11 diff --git ql/src/test/results/clientpositive/perf/query87.q.out ql/src/test/results/clientpositive/perf/query87.q.out index 58a33d9..88dc197 100644 --- ql/src/test/results/clientpositive/perf/query87.q.out +++ ql/src/test/results/clientpositive/perf/query87.q.out @@ -99,9 +99,9 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_78] PartitionCols:_col0 - Select Operator [SEL_73] (rows=80000000 width=860) + Select Operator [SEL_8] (rows=80000000 width=860) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_128] (rows=80000000 width=860) + Filter Operator [FIL_122] (rows=80000000 width=860) predicate:c_customer_sk is not null TableScan [TS_6] (rows=80000000 width=860) default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"] @@ -113,9 +113,9 @@ Stage-0 <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_75] PartitionCols:_col0 - Select Operator [SEL_70] (rows=8116 width=1119) + Select Operator [SEL_5] (rows=8116 width=1119) Output:["_col0","_col1"] - Filter Operator [FIL_127] (rows=8116 width=1119) + Filter Operator [FIL_121] (rows=8116 width=1119) predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"] @@ -169,11 +169,7 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_39] PartitionCols:_col0 - Select Operator [SEL_34] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_125] (rows=80000000 width=860) - predicate:c_customer_sk is not null - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:_col1 @@ -182,11 +178,7 @@ Stage-0 <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_36] PartitionCols:_col0 - Select Operator [SEL_31] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_124] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_35] PartitionCols:_col0 @@ -219,11 +211,7 @@ Stage-0 <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_8] (rows=80000000 width=860) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_122] (rows=80000000 width=860) - predicate:c_customer_sk is not null - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col1 @@ -232,11 +220,7 @@ Stage-0 <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_10] PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_121] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_9] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query88.q.out ql/src/test/results/clientpositive/perf/query88.q.out index 18d0a77..6bad30c 100644 --- ql/src/test/results/clientpositive/perf/query88.q.out +++ ql/src/test/results/clientpositive/perf/query88.q.out @@ -243,9 +243,9 @@ Stage-0 <-Map 37 [SIMPLE_EDGE] SHUFFLE [RS_45] PartitionCols:_col0 - Select Operator [SEL_37] (rows=852 width=1910) + Select Operator [SEL_11] (rows=852 width=1910) Output:["_col0"] - Filter Operator [FIL_298] (rows=852 width=1910) + Filter Operator [FIL_294] (rows=852 width=1910) predicate:((s_store_name = 'ese') and s_store_sk is not null) TableScan [TS_9] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"] @@ -257,9 +257,9 @@ Stage-0 <-Map 36 [SIMPLE_EDGE] SHUFFLE [RS_42] PartitionCols:_col0 - Select Operator [SEL_34] (rows=3600 width=107) + Select Operator [SEL_8] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_297] (rows=3600 width=107) + Filter Operator [FIL_293] (rows=3600 width=107) predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) TableScan [TS_6] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"] @@ -271,9 +271,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:_col0 - Select Operator [SEL_28] (rows=575995635 width=88) + Select Operator [SEL_2] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_295] (rows=575995635 width=88) + Filter Operator [FIL_291] (rows=575995635 width=88) predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"] @@ -299,11 +299,7 @@ Stage-0 <-Map 37 [SIMPLE_EDGE] SHUFFLE [RS_71] PartitionCols:_col0 - Select Operator [SEL_63] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_302] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_70] PartitionCols:_col2 @@ -312,11 +308,7 @@ Stage-0 <-Map 36 [SIMPLE_EDGE] SHUFFLE [RS_68] PartitionCols:_col0 - Select Operator [SEL_60] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_301] (rows=3600 width=107) - predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_67] PartitionCols:_col1 @@ -325,11 +317,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_64] PartitionCols:_col0 - Select Operator [SEL_54] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_299] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 35 [SIMPLE_EDGE] SHUFFLE [RS_65] PartitionCols:_col0 @@ -351,11 +339,7 @@ Stage-0 <-Map 37 [SIMPLE_EDGE] SHUFFLE [RS_97] PartitionCols:_col0 - Select Operator [SEL_89] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_306] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 16 [SIMPLE_EDGE] SHUFFLE [RS_96] PartitionCols:_col2 @@ -364,11 +348,7 @@ Stage-0 <-Map 36 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0 - Select Operator [SEL_86] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_305] (rows=3600 width=107) - predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_93] PartitionCols:_col1 @@ -377,11 +357,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_90] PartitionCols:_col0 - Select Operator [SEL_80] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_303] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 35 [SIMPLE_EDGE] SHUFFLE [RS_91] PartitionCols:_col0 @@ -403,11 +379,7 @@ Stage-0 <-Map 37 [SIMPLE_EDGE] SHUFFLE [RS_123] PartitionCols:_col0 - Select Operator [SEL_115] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_310] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_122] PartitionCols:_col2 @@ -416,11 +388,7 @@ Stage-0 <-Map 36 [SIMPLE_EDGE] SHUFFLE [RS_120] PartitionCols:_col0 - Select Operator [SEL_112] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_309] (rows=3600 width=107) - predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 19 [SIMPLE_EDGE] SHUFFLE [RS_119] PartitionCols:_col1 @@ -429,11 +397,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_116] PartitionCols:_col0 - Select Operator [SEL_106] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_307] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 35 [SIMPLE_EDGE] SHUFFLE [RS_117] PartitionCols:_col0 @@ -455,11 +419,7 @@ Stage-0 <-Map 37 [SIMPLE_EDGE] SHUFFLE [RS_149] PartitionCols:_col0 - Select Operator [SEL_141] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_314] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 24 [SIMPLE_EDGE] SHUFFLE [RS_148] PartitionCols:_col2 @@ -468,11 +428,7 @@ Stage-0 <-Map 36 [SIMPLE_EDGE] SHUFFLE [RS_146] PartitionCols:_col0 - Select Operator [SEL_138] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_313] (rows=3600 width=107) - predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 23 [SIMPLE_EDGE] SHUFFLE [RS_145] PartitionCols:_col1 @@ -481,11 +437,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_142] PartitionCols:_col0 - Select Operator [SEL_132] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_311] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 35 [SIMPLE_EDGE] SHUFFLE [RS_143] PartitionCols:_col0 @@ -507,11 +459,7 @@ Stage-0 <-Map 37 [SIMPLE_EDGE] SHUFFLE [RS_175] PartitionCols:_col0 - Select Operator [SEL_167] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_318] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 28 [SIMPLE_EDGE] SHUFFLE [RS_174] PartitionCols:_col2 @@ -520,11 +468,7 @@ Stage-0 <-Map 36 [SIMPLE_EDGE] SHUFFLE [RS_172] PartitionCols:_col0 - Select Operator [SEL_164] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_317] (rows=3600 width=107) - predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 27 [SIMPLE_EDGE] SHUFFLE [RS_171] PartitionCols:_col1 @@ -533,11 +477,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_168] PartitionCols:_col0 - Select Operator [SEL_158] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_315] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 35 [SIMPLE_EDGE] SHUFFLE [RS_169] PartitionCols:_col0 @@ -559,11 +499,7 @@ Stage-0 <-Map 37 [SIMPLE_EDGE] SHUFFLE [RS_201] PartitionCols:_col0 - Select Operator [SEL_193] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_322] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 32 [SIMPLE_EDGE] SHUFFLE [RS_200] PartitionCols:_col2 @@ -572,11 +508,7 @@ Stage-0 <-Map 36 [SIMPLE_EDGE] SHUFFLE [RS_198] PartitionCols:_col0 - Select Operator [SEL_190] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_321] (rows=3600 width=107) - predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 31 [SIMPLE_EDGE] SHUFFLE [RS_197] PartitionCols:_col1 @@ -585,11 +517,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_194] PartitionCols:_col0 - Select Operator [SEL_184] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_319] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 35 [SIMPLE_EDGE] SHUFFLE [RS_195] PartitionCols:_col0 @@ -611,11 +539,7 @@ Stage-0 <-Map 37 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 - Select Operator [SEL_11] (rows=852 width=1910) - Output:["_col0"] - Filter Operator [FIL_294] (rows=852 width=1910) - predicate:((s_store_name = 'ese') and s_store_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col2 @@ -624,11 +548,7 @@ Stage-0 <-Map 36 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_8] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_293] (rows=3600 width=107) - predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) - Please refer to the previous TableScan [TS_6] + Please refer to the previous Select Operator [SEL_8] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col1 @@ -637,11 +557,7 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_291] (rows=575995635 width=88) - predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 35 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/perf/query90.q.out ql/src/test/results/clientpositive/perf/query90.q.out index b3468ec..d63ce1b 100644 --- ql/src/test/results/clientpositive/perf/query90.q.out +++ ql/src/test/results/clientpositive/perf/query90.q.out @@ -84,9 +84,9 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_45] PartitionCols:_col0 - Select Operator [SEL_37] (rows=3600 width=107) + Select Operator [SEL_11] (rows=3600 width=107) Output:["_col0"] - Filter Operator [FIL_85] (rows=3600 width=107) + Filter Operator [FIL_81] (rows=3600 width=107) predicate:((hd_dep_count = 8) and hd_demo_sk is not null) TableScan [TS_9] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] @@ -112,18 +112,18 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:_col2 - Select Operator [SEL_28] (rows=144002668 width=135) + Select Operator [SEL_2] (rows=144002668 width=135) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_82] (rows=144002668 width=135) + Filter Operator [FIL_78] (rows=144002668 width=135) predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null) TableScan [TS_0] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"] <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_39] PartitionCols:_col0 - Select Operator [SEL_31] (rows=511 width=585) + Select Operator [SEL_5] (rows=511 width=585) Output:["_col0"] - Filter Operator [FIL_83] (rows=511 width=585) + Filter Operator [FIL_79] (rows=511 width=585) predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) TableScan [TS_3] (rows=4602 width=585) default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk","wp_char_count"] @@ -140,11 +140,7 @@ Stage-0 <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 - Select Operator [SEL_11] (rows=3600 width=107) - Output:["_col0"] - Filter Operator [FIL_81] (rows=3600 width=107) - predicate:((hd_dep_count = 8) and hd_demo_sk is not null) - Please refer to the previous TableScan [TS_9] + Please refer to the previous Select Operator [SEL_11] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col1 @@ -166,17 +162,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col2 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_78] (rows=144002668 width=135) - predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_5] (rows=511 width=585) - Output:["_col0"] - Filter Operator [FIL_79] (rows=511 width=585) - predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] diff --git ql/src/test/results/clientpositive/perf/query92.q.out ql/src/test/results/clientpositive/perf/query92.q.out index ec4fbb9..761753f 100644 --- ql/src/test/results/clientpositive/perf/query92.q.out +++ ql/src/test/results/clientpositive/perf/query92.q.out @@ -147,17 +147,9 @@ Stage-0 <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_12] PartitionCols:_col0 - Select Operator [SEL_8] (rows=144002668 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_58] (rows=144002668 width=135) - predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) - Please refer to the previous TableScan [TS_0] + Please refer to the previous Select Operator [SEL_2] <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 - Select Operator [SEL_11] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_59] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-03-18 00:00:00.0 AND 1998-06-16 01:00:00.0 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] diff --git ql/src/test/results/clientpositive/perf/query94.q.out ql/src/test/results/clientpositive/perf/query94.q.out index c5fc9e7..146ad1e 100644 --- ql/src/test/results/clientpositive/perf/query94.q.out +++ ql/src/test/results/clientpositive/perf/query94.q.out @@ -1,4 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[113][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 18' is a cross product +Warning: Shuffle Join MERGEJOIN[107][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 17' is a cross product PREHOOK: query: explain select count(distinct ws_order_number) as `order count` @@ -58,182 +58,172 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 14 <- Map 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Map 15 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Map 21 (CUSTOM_SIMPLE_EDGE), Map 22 (CUSTOM_SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE) -Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 14 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 17 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) +Reducer 13 <- Map 12 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) +Reducer 16 <- Reducer 15 (SIMPLE_EDGE) +Reducer 17 <- Map 14 (CUSTOM_SIMPLE_EDGE), Map 19 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Map 21 (CUSTOM_SIMPLE_EDGE) +Reducer 18 <- Reducer 17 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) +Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 6 <- Reducer 16 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Reducer 8 <- Reducer 7 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 + Reducer 8 File Output Operator [FS_74] Limit [LIM_72] (rows=1 width=344) Number of rows:100 Select Operator [SEL_71] (rows=1 width=344) Output:["_col0","_col1","_col2"] - <-Reducer 8 [SIMPLE_EDGE] + <-Reducer 7 [SIMPLE_EDGE] SHUFFLE [RS_70] Select Operator [SEL_69] (rows=1 width=344) Output:["_col1","_col2","_col3"] - Group By Operator [GBY_112] (rows=1 width=344) - Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] - <-Reducer 7 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_111] - Group By Operator [GBY_110] (rows=1 width=344) - Output:["_col0","_col1","_col2"],aggregations:["count(_col0)","sum(_col1)","sum(_col2)"] - Group By Operator [GBY_109] (rows=1395035081047425024 width=1) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_108] - PartitionCols:_col0 - Group By Operator [GBY_107] (rows=1395035081047425024 width=1) - Output:["_col0","_col2","_col3"],aggregations:["sum(_col5)","sum(_col6)"],keys:_col4 - Select Operator [SEL_65] (rows=1395035081047425024 width=1) - Output:["_col4","_col5","_col6"] - Filter Operator [FIL_64] (rows=1395035081047425024 width=1) - predicate:_col16 is null - Select Operator [SEL_63] (rows=2790070162094850048 width=1) - Output:["_col4","_col5","_col6","_col16"] - Merge Join Operator [MERGEJOIN_119] (rows=2790070162094850048 width=1) - Conds:RS_60._col3, _col4=RS_61._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_61] - PartitionCols:_col0, _col1 - Group By Operator [GBY_46] (rows=2536427365110644736 width=1) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0, _col1 - Group By Operator [GBY_44] (rows=5072854730221289472 width=1) - Output:["_col0","_col1"],keys:_col2, _col3 - Select Operator [SEL_43] (rows=5072854730221289472 width=1) - Output:["_col2","_col3"] - Filter Operator [FIL_42] (rows=5072854730221289472 width=1) - predicate:(_col2 <> _col0) - Merge Join Operator [MERGEJOIN_117] (rows=5072854730221289472 width=1) - Conds:RS_39._col1=RS_40._col1(Inner),Output:["_col0","_col2","_col3"] - <-Map 15 [SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_39] - PartitionCols:_col1 - Select Operator [SEL_20] (rows=144002668 width=135) - Output:["_col0","_col1"] - TableScan [TS_19] (rows=144002668 width=135) - default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1 - Select Operator [SEL_38] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"] - Group By Operator [GBY_37] (rows=4611686018427387903 width=1) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Reducer 18 [SIMPLE_EDGE] - SHUFFLE [RS_36] - PartitionCols:_col0, _col1 - Group By Operator [GBY_35] (rows=9223372036854775807 width=1) - Output:["_col0","_col1"],keys:_col4, _col3 - Merge Join Operator [MERGEJOIN_113] (rows=9223372036854775807 width=1) - Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"] - <-Map 15 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_32] - Select Operator [SEL_28] (rows=144002668 width=135) - Output:["_col0","_col1"] - Please refer to the previous TableScan [TS_19] - <-Map 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_29] - Select Operator [SEL_22] (rows=73049 width=4) - TableScan [TS_21] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE - <-Map 21 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_30] - Select Operator [SEL_24] (rows=84 width=4) - TableScan [TS_23] (rows=84 width=1850) - default@web_site,web_site,Tbl:COMPLETE,Col:COMPLETE - <-Map 22 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_31] - Select Operator [SEL_26] (rows=40000000 width=4) - TableScan [TS_25] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col3, _col4 - Merge Join Operator [MERGEJOIN_118] (rows=210834322 width=135) - Conds:RS_57._col4=RS_58._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_58] + Group By Operator [GBY_68] (rows=1 width=344) + Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"] + <-Reducer 6 [SIMPLE_EDGE] + SHUFFLE [RS_67] + Group By Operator [GBY_66] (rows=1395035081047425024 width=1) + Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col4)","sum(_col5)","sum(_col6)"],keys:_col4 + Select Operator [SEL_65] (rows=1395035081047425024 width=1) + Output:["_col4","_col5","_col6"] + Filter Operator [FIL_64] (rows=1395035081047425024 width=1) + predicate:_col16 is null + Select Operator [SEL_63] (rows=2790070162094850048 width=1) + Output:["_col4","_col5","_col6","_col16"] + Merge Join Operator [MERGEJOIN_113] (rows=2790070162094850048 width=1) + Conds:RS_60._col3, _col4=RS_61._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"] + <-Reducer 16 [SIMPLE_EDGE] + SHUFFLE [RS_61] + PartitionCols:_col0, _col1 + Group By Operator [GBY_46] (rows=2536427365110644736 width=1) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Reducer 15 [SIMPLE_EDGE] + SHUFFLE [RS_45] + PartitionCols:_col0, _col1 + Group By Operator [GBY_44] (rows=5072854730221289472 width=1) + Output:["_col0","_col1"],keys:_col2, _col3 + Select Operator [SEL_43] (rows=5072854730221289472 width=1) + Output:["_col2","_col3"] + Filter Operator [FIL_42] (rows=5072854730221289472 width=1) + predicate:(_col2 <> _col0) + Merge Join Operator [MERGEJOIN_111] (rows=5072854730221289472 width=1) + Conds:RS_39._col1=RS_40._col1(Inner),Output:["_col0","_col2","_col3"] + <-Map 14 [SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_39] + PartitionCols:_col1 + Select Operator [SEL_20] (rows=144002668 width=135) + Output:["_col0","_col1"] + TableScan [TS_19] (rows=144002668 width=135) + default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] + <-Reducer 18 [SIMPLE_EDGE] + SHUFFLE [RS_40] + PartitionCols:_col1 + Select Operator [SEL_38] (rows=4611686018427387903 width=1) + Output:["_col0","_col1"] + Group By Operator [GBY_37] (rows=4611686018427387903 width=1) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Reducer 17 [SIMPLE_EDGE] + SHUFFLE [RS_36] + PartitionCols:_col0, _col1 + Group By Operator [GBY_35] (rows=9223372036854775807 width=1) + Output:["_col0","_col1"],keys:_col4, _col3 + Merge Join Operator [MERGEJOIN_107] (rows=9223372036854775807 width=1) + Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"] + <-Map 14 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_32] + Please refer to the previous Select Operator [SEL_20] + <-Map 19 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_29] + Select Operator [SEL_22] (rows=73049 width=4) + TableScan [TS_21] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE + <-Map 20 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_30] + Select Operator [SEL_24] (rows=84 width=4) + TableScan [TS_23] (rows=84 width=1850) + default@web_site,web_site,Tbl:COMPLETE,Col:COMPLETE + <-Map 21 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_31] + Select Operator [SEL_26] (rows=40000000 width=4) + TableScan [TS_25] (rows=40000000 width=1014) + default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_60] + PartitionCols:_col3, _col4 + Merge Join Operator [MERGEJOIN_112] (rows=210834322 width=135) + Conds:RS_57._col4=RS_58._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"] + <-Reducer 13 [SIMPLE_EDGE] + SHUFFLE [RS_58] + PartitionCols:_col0 + Select Operator [SEL_18] (rows=7199233 width=92) + Output:["_col0","_col1"] + Group By Operator [GBY_17] (rows=7199233 width=92) + Output:["_col0"],keys:KEY._col0 + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_18] (rows=7199233 width=92) - Output:["_col0","_col1"] - Group By Operator [GBY_17] (rows=7199233 width=92) - Output:["_col0"],keys:KEY._col0 - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_16] + Group By Operator [GBY_15] (rows=14398467 width=92) + Output:["_col0"],keys:wr_order_number + Filter Operator [FIL_104] (rows=14398467 width=92) + predicate:wr_order_number is not null + TableScan [TS_12] (rows=14398467 width=92) + default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"] + <-Reducer 4 [SIMPLE_EDGE] + SHUFFLE [RS_57] + PartitionCols:_col4 + Merge Join Operator [MERGEJOIN_110] (rows=191667562 width=135) + Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6"] + <-Map 11 [SIMPLE_EDGE] + SHUFFLE [RS_55] + PartitionCols:_col0 + Select Operator [SEL_11] (rows=42 width=1850) + Output:["_col0"] + Filter Operator [FIL_103] (rows=42 width=1850) + predicate:((web_company_name = 'pri') and web_site_sk is not null) + TableScan [TS_9] (rows=84 width=1850) + default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_54] + PartitionCols:_col2 + Merge Join Operator [MERGEJOIN_109] (rows=174243235 width=135) + Conds:RS_51._col1=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"] + <-Map 10 [SIMPLE_EDGE] + SHUFFLE [RS_52] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=20000000 width=1014) + Output:["_col0"] + Filter Operator [FIL_102] (rows=20000000 width=1014) + predicate:((ca_state = 'TX') and ca_address_sk is not null) + TableScan [TS_6] (rows=40000000 width=1014) + default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_51] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_108] (rows=158402938 width=135) + Conds:RS_48._col0=RS_49._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_48] PartitionCols:_col0 - Group By Operator [GBY_15] (rows=14398467 width=92) - Output:["_col0"],keys:wr_order_number - Filter Operator [FIL_104] (rows=14398467 width=92) - predicate:wr_order_number is not null - TableScan [TS_12] (rows=14398467 width=92) - default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_116] (rows=191667562 width=135) - Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_55] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=42 width=1850) - Output:["_col0"] - Filter Operator [FIL_103] (rows=42 width=1850) - predicate:((web_company_name = 'pri') and web_site_sk is not null) - TableScan [TS_9] (rows=84 width=1850) - default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_54] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_115] (rows=174243235 width=135) - Conds:RS_51._col1=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_102] (rows=20000000 width=1014) - predicate:((ca_state = 'TX') and ca_address_sk is not null) - TableScan [TS_6] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_114] (rows=158402938 width=135) - Conds:RS_48._col0=RS_49._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Filter Operator [FIL_100] (rows=144002668 width=135) - predicate:(ws_ship_date_sk is not null and ws_ship_addr_sk is not null and ws_web_site_sk is not null) - TableScan [TS_0] (rows=144002668 width=135) - default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_warehouse_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_101] (rows=8116 width=1119) - predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] + Select Operator [SEL_2] (rows=144002668 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + Filter Operator [FIL_100] (rows=144002668 width=135) + predicate:(ws_ship_date_sk is not null and ws_ship_addr_sk is not null and ws_web_site_sk is not null) + TableScan [TS_0] (rows=144002668 width=135) + default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_warehouse_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"] + <-Map 9 [SIMPLE_EDGE] + SHUFFLE [RS_49] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=8116 width=1119) + Output:["_col0"] + Filter Operator [FIL_101] (rows=8116 width=1119) + predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null) + TableScan [TS_3] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] diff --git ql/src/test/results/clientpositive/perf/query95.q.out ql/src/test/results/clientpositive/perf/query95.q.out index 332bef8..28db5a6 100644 --- ql/src/test/results/clientpositive/perf/query95.q.out +++ ql/src/test/results/clientpositive/perf/query95.q.out @@ -172,19 +172,11 @@ Stage-0 <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_33] PartitionCols:_col1 - Select Operator [SEL_29] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_112] (rows=144002668 width=135) - predicate:ws_order_number is not null - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_34] PartitionCols:_col1 - Select Operator [SEL_32] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_113] (rows=144002668 width=135) - predicate:ws_order_number is not null - Please refer to the previous TableScan [TS_15] + Please refer to the previous Select Operator [SEL_17] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_58] PartitionCols:_col3 diff --git ql/src/test/results/clientpositive/perf/query97.q.out ql/src/test/results/clientpositive/perf/query97.q.out index 2d5129a..51dd341 100644 --- ql/src/test/results/clientpositive/perf/query97.q.out +++ ql/src/test/results/clientpositive/perf/query97.q.out @@ -119,11 +119,7 @@ Stage-0 <-Map 6 [SIMPLE_EDGE] SHUFFLE [RS_21] PartitionCols:_col0 - Select Operator [SEL_19] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_46] (rows=8116 width=1119) - predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null) - Please refer to the previous TableScan [TS_3] + Please refer to the previous Select Operator [SEL_5] <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_20] PartitionCols:_col0 diff --git ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out index ec87daa..9ced7ad 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out @@ -86,9 +86,9 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_44] PartitionCols:_col1 - Select Operator [SEL_39] (rows=25/25 width=175) + Select Operator [SEL_14] (rows=25/25 width=175) Output:["_col0","_col1"] - Filter Operator [FIL_77] (rows=25/25 width=175) + Filter Operator [FIL_73] (rows=25/25 width=175) predicate:(key is not null and value is not null) TableScan [TS_12] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -142,11 +142,7 @@ Stage-0 <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col1 - Select Operator [SEL_14] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_73] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col1 @@ -276,9 +272,9 @@ Stage-0 <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_105] PartitionCols:_col0 - Select Operator [SEL_100] (rows=500/500 width=178) + Select Operator [SEL_17] (rows=500/500 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_156] (rows=500/500 width=178) + Filter Operator [FIL_145] (rows=500/500 width=178) predicate:key is not null TableScan [TS_15] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -290,9 +286,9 @@ Stage-0 <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_102] PartitionCols:_col1 - Select Operator [SEL_97] (rows=25/25 width=175) + Select Operator [SEL_14] (rows=25/25 width=175) Output:["_col0","_col1"] - Filter Operator [FIL_155] (rows=25/25 width=175) + Filter Operator [FIL_144] (rows=25/25 width=175) predicate:(key is not null and value is not null) TableScan [TS_12] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -386,11 +382,7 @@ Stage-0 <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_53] PartitionCols:_col1 - Select Operator [SEL_48] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_149] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 21 [SIMPLE_EDGE] SHUFFLE [RS_52] PartitionCols:_col1 @@ -444,11 +436,7 @@ Stage-0 <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col0 - Select Operator [SEL_17] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_145] (rows=500/500 width=178) - predicate:key is not null - Please refer to the previous TableScan [TS_15] + Please refer to the previous Select Operator [SEL_17] <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_21] PartitionCols:_col2 @@ -457,11 +445,7 @@ Stage-0 <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col1 - Select Operator [SEL_14] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_144] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col1 @@ -1280,11 +1264,7 @@ Stage-5 <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_66] PartitionCols:_col0 - Select Operator [SEL_48] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_113] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_8] + Please refer to the previous Select Operator [SEL_10] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_67] PartitionCols:_col0 @@ -1478,9 +1458,9 @@ Stage-5 <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_101] PartitionCols:_col0 - Select Operator [SEL_67] (rows=25/25 width=175) + Select Operator [SEL_14] (rows=25/25 width=175) Output:["_col0","_col1"] - Filter Operator [FIL_153] (rows=25/25 width=175) + Filter Operator [FIL_146] (rows=25/25 width=175) predicate:(key is not null and value is not null) TableScan [TS_12] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -1655,11 +1635,7 @@ Stage-5 <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col1 - Select Operator [SEL_14] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_146] (rows=25/25 width=175) - predicate:(key is not null and value is not null) - Please refer to the previous TableScan [TS_12] + Please refer to the previous Select Operator [SEL_14] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_18] PartitionCols:_col1