diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index be83489cb3..1dc7501291 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1695,6 +1695,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_SHARED_WORK_OPTIMIZATION("hive.optimize.shared.work", true, "Whether to enable shared work optimizer. The optimizer finds scan operator over the same table\n" + "and follow-up operators in the query plan and merges them if they meet some preconditions. Tez only."), + HIVE_SHARED_WORK_EXTENDED_OPTIMIZATION("hive.optimize.shared.work.extended", true, + "Whether to enable shared work extended optimizer. The optimizer tries to merge equal operators\n" + + "after a work boundary after shared work optimizer has been executed. Requires hive.optimize.shared.work\n" + + "to be set to true. Tez only."), HIVE_COMBINE_EQUIVALENT_WORK_OPTIMIZATION("hive.combine.equivalent.work.optimization", true, "Whether to " + "combine equivalent work objects during physical optimization.\n This optimization looks for equivalent " + "work objects and combines them if they meet certain preconditions. Spark only."), diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 9121ca8d1e..355818ea8d 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -629,6 +629,7 @@ minillaplocal.query.files=\ semijoin6.q,\ semijoin7.q,\ semijoin_hint.q,\ + sharedworkext.q,\ smb_cache.q,\ special_character_in_tabnames_1.q,\ sqlmerge.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java index d4ddb75c90..a2a7871ee8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java @@ -36,6 +36,7 @@ import java.util.TreeMap; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator; import org.apache.hadoop.hive.ql.exec.DummyStoreOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; @@ -79,6 +80,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import com.google.common.collect.Multiset; +import com.google.common.collect.Sets; import com.google.common.collect.TreeMultiset; /** @@ -102,8 +104,8 @@ * | | / \ * Op Op Op Op * - *

A limitation in the current implementation is that the optimizer does not - * go beyond a work boundary. + *

If the extended version of the optimizer is enabled, it can go beyond + * a work boundary to find reutilization opportunities. * *

The optimization only works with the Tez execution engine. */ @@ -146,13 +148,13 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { String tableName = tablePair.getKey(); for (TableScanOperator discardableTsOp : tableNameToOps.get(tableName)) { if (removedOps.contains(discardableTsOp)) { - LOG.debug("Skip {} as it has been already removed", discardableTsOp); + LOG.debug("Skip {} as it has already been removed", discardableTsOp); continue; } Collection prevTsOps = existingOps.get(tableName); for (TableScanOperator retainableTsOp : prevTsOps) { if (removedOps.contains(retainableTsOp)) { - LOG.debug("Skip {} as it has been already removed", retainableTsOp); + LOG.debug("Skip {} as it has already been removed", retainableTsOp); continue; } @@ -167,7 +169,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // Secondly, we extract information about the part of the tree that can be merged // as well as some structural information (memory consumption) that needs to be // used to determined whether the merge can happen - SharedResult sr = extractSharedOptimizationInfo( + SharedResult sr = extractSharedOptimizationInfoForRoot( pctx, optimizerCache, retainableTsOp, discardableTsOp); // It seems these two operators can be merged. @@ -328,6 +330,194 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { LOG.debug("After SharedWorkOptimizer:\n" + Operator.toString(pctx.getTopOps().values())); } + if(pctx.getConf().getBoolVar(ConfVars.HIVE_SHARED_WORK_EXTENDED_OPTIMIZATION)) { + // Gather RS operators that 1) belong to root works, i.e., works containing TS operators, + // and 2) share the same input operator. + // These will be the first target for extended shared work optimization + Multimap, ReduceSinkOperator> parentToRsOps = HashMultimap.create(); + for (Entry e : topOps.entrySet()) { + for (Operator op : findWorkOperators(optimizerCache, e.getValue())) { + if (op instanceof ReduceSinkOperator && op.getParentOperators().get(0).getNumChild() > 1) { + parentToRsOps.put(op.getParentOperators().get(0), (ReduceSinkOperator) op); + } + } + } + + while (!parentToRsOps.isEmpty()) { + // As above, we enforce a certain order when we do the reutilization. + // In particular, we use size of data in RS x number of uses. + List, Long>> sortedRSGroups = rankOpsByAccumulatedSize(parentToRsOps.keySet()); + LOG.debug("Sorted operators by size: {}", sortedRSGroups); + + // Execute extended optimization + // For each RS, check whether other RS in same work could be merge into this one. + // If they are merged, RS operators in the resulting work will be considered + // mergeable in next loop iteration. + Multimap, ReduceSinkOperator> existingRsOps = ArrayListMultimap.create(); + removedOps = new HashSet<>(); + for (Entry, Long> rsGroupInfo : sortedRSGroups) { + Operator rsParent = rsGroupInfo.getKey(); + for (ReduceSinkOperator discardableRsOp : parentToRsOps.get(rsParent)) { + if (removedOps.contains(discardableRsOp)) { + LOG.debug("Skip {} as it has already been removed", discardableRsOp); + continue; + } + Collection otherRsOps = existingRsOps.get(rsParent); + for (ReduceSinkOperator retainableRsOp : otherRsOps) { + if (removedOps.contains(retainableRsOp)) { + LOG.debug("Skip {} as it has already been removed", retainableRsOp); + continue; + } + + // First we quickly check if the two RS operators can actually be merged. + // We already know that these two RS operators have the same parent, but + // we need to check whether both RS are actually equal. Further, we check + // whether their child is also equal. If any of these conditions are not + // met, we are not going to try to merge. + boolean mergeable = compareOperator(pctx, retainableRsOp, discardableRsOp) && + compareOperator(pctx, retainableRsOp.getChildOperators().get(0), discardableRsOp.getChildOperators().get(0)); + if (!mergeable) { + // Skip + LOG.debug("{} and {} cannot be merged", retainableRsOp, discardableRsOp); + continue; + } + + // Secondly, we extract information about the part of the tree that can be merged + // as well as some structural information (memory consumption) that needs to be + // used to determined whether the merge can happen + Operator retainableRsOpChild = retainableRsOp.getChildOperators().get(0); + Operator discardableRsOpChild = discardableRsOp.getChildOperators().get(0); + SharedResult sr = extractSharedOptimizationInfo( + pctx, optimizerCache, retainableRsOp, discardableRsOp, + retainableRsOpChild, discardableRsOpChild); + + // It seems these two operators can be merged. + // Check that plan meets some preconditions before doing it. + // In particular, in the presence of map joins in the upstream plan: + // - we cannot exceed the noconditional task size, and + // - if we already merged the big table, we cannot merge the broadcast + // tables. + if (sr.retainableOps.isEmpty() || !validPreConditions(pctx, optimizerCache, sr)) { + // Skip + LOG.debug("{} and {} do not meet preconditions", retainableRsOp, discardableRsOp); + continue; + } + + // We can merge + Operator lastRetainableOp = sr.retainableOps.get(sr.retainableOps.size() - 1); + Operator lastDiscardableOp = sr.discardableOps.get(sr.discardableOps.size() - 1); + if (lastDiscardableOp.getNumChild() != 0) { + List> allChildren = + Lists.newArrayList(lastDiscardableOp.getChildOperators()); + for (Operator op : allChildren) { + lastDiscardableOp.getChildOperators().remove(op); + op.replaceParent(lastDiscardableOp, lastRetainableOp); + lastRetainableOp.getChildOperators().add(op); + } + } + + LOG.debug("Merging subtree starting at {} into subtree starting at {}", + discardableRsOp, retainableRsOp); + + // First we remove the input operators of the expression that + // we are going to eliminate + for (Operator op : sr.discardableInputOps) { + OperatorUtils.removeOperator(op); + optimizerCache.removeOp(op); + removedOps.add(op); + // Remove DPP predicates + if (op instanceof ReduceSinkOperator) { + SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op); + if (sjbi != null && !sr.discardableOps.contains(sjbi.getTsOp()) && + !sr.discardableInputOps.contains(sjbi.getTsOp())) { + GenTezUtils.removeSemiJoinOperator( + pctx, (ReduceSinkOperator) op, sjbi.getTsOp()); + } + } else if (op instanceof AppMasterEventOperator) { + DynamicPruningEventDesc dped = (DynamicPruningEventDesc) op.getConf(); + if (!sr.discardableOps.contains(dped.getTableScan()) && + !sr.discardableInputOps.contains(dped.getTableScan())) { + GenTezUtils.removeSemiJoinOperator( + pctx, (AppMasterEventOperator) op, dped.getTableScan()); + } + } + LOG.debug("Input operator removed: {}", op); + } + // We remove the discardable RS operator + OperatorUtils.removeOperator(discardableRsOp); + optimizerCache.removeOp(discardableRsOp); + removedOps.add(discardableRsOp); + LOG.debug("Operator removed: {}", discardableRsOp); + // Then we merge the operators of the works we are going to merge + optimizerCache.removeOpAndCombineWork(discardableRsOpChild, retainableRsOpChild); + // Finally we remove the rest of the expression from the tree + for (Operator op : sr.discardableOps) { + OperatorUtils.removeOperator(op); + optimizerCache.removeOp(op); + removedOps.add(op); + LOG.debug("Operator removed: {}", op); + } + + break; + } + + if (removedOps.contains(discardableRsOp)) { + // This operator has been removed, remove it from the list of existing operators + existingRsOps.remove(rsParent, discardableRsOp); + } else { + // This operator has not been removed, include it in the list of existing operators + existingRsOps.put(rsParent, discardableRsOp); + } + } + } + + // We gather the operators that will be used for next iteration of extended optimization (if any) + parentToRsOps = HashMultimap.create(); + for (Entry, ReduceSinkOperator> e : existingRsOps.entries()) { + for (Operator op : findWorkOperators(optimizerCache, e.getValue().getChildOperators().get(0))) { + if (op instanceof ReduceSinkOperator && op.getParentOperators().get(0).getNumChild() > 1) { + parentToRsOps.put(op.getParentOperators().get(0), (ReduceSinkOperator) op); + } + } + } + } + + // Remove unused table scan operators + it = topOps.entrySet().iterator(); + while (it.hasNext()) { + Entry e = it.next(); + if (e.getValue().getNumChild() == 0) { + it.remove(); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("After SharedWorkExtendedOptimizer:\n" + Operator.toString(pctx.getTopOps().values())); + } + } + + // If we are running tests, we are going to verify that the contents of the cache + // correspond with the contents of the plan, and otherwise we fail. + // This check always run when we are running in test mode, independently on whether + // we use the basic or the extended version of the optimizer. + if (pctx.getConf().getBoolVar(ConfVars.HIVE_IN_TEST)) { + Set> visited = new HashSet<>(); + it = topOps.entrySet().iterator(); + while (it.hasNext()) { + Entry e = it.next(); + for (Operator op : OperatorUtils.findOperators(e.getValue(), Operator.class)) { + if (!visited.contains(op)) { + if (!findWorkOperators(optimizerCache, op).equals( + findWorkOperators(op, new HashSet>()))) { + throw new SemanticException("Error in shared work optimizer: operator cache contents" + + "and actual plan differ"); + } + visited.add(op); + } + } + } + } + return pctx; } @@ -402,6 +592,25 @@ public int compare(Map.Entry o1, Map.Entry o2) { return sortedTables; } + private static List, Long>> rankOpsByAccumulatedSize(Set> opsSet) { + Map, Long> opToTotalSize = new HashMap<>(); + for (Operator op : opsSet) { + long size = op.getStatistics() != null ? + op.getStatistics().getDataSize() : 0L; + opToTotalSize.put(op, + StatsUtils.safeMult(op.getChildOperators().size(), size)); + } + List, Long>> sortedOps = + new LinkedList<>(opToTotalSize.entrySet()); + Collections.sort(sortedOps, Collections.reverseOrder( + new Comparator, Long>>() { + public int compare(Map.Entry, Long> o1, Map.Entry, Long> o2) { + return (o1.getValue()).compareTo(o2.getValue()); + } + })); + return sortedOps; + } + private static boolean areMergeable(ParseContext pctx, SharedWorkOptimizerCache optimizerCache, TableScanOperator tsOp1, TableScanOperator tsOp2) throws SemanticException { // First we check if the two table scan operators can actually be merged @@ -486,12 +695,12 @@ private static boolean areMergeable(ParseContext pctx, SharedWorkOptimizerCache return true; } - private static SharedResult extractSharedOptimizationInfo(ParseContext pctx, + private static SharedResult extractSharedOptimizationInfoForRoot(ParseContext pctx, SharedWorkOptimizerCache optimizerCache, TableScanOperator retainableTsOp, TableScanOperator discardableTsOp) throws SemanticException { - Set> retainableOps = new LinkedHashSet<>(); - Set> discardableOps = new LinkedHashSet<>(); + LinkedHashSet> retainableOps = new LinkedHashSet<>(); + LinkedHashSet> discardableOps = new LinkedHashSet<>(); Set> discardableInputOps = new HashSet<>(); long dataSize = 0l; long maxDataSize = 0l; @@ -545,6 +754,36 @@ private static SharedResult extractSharedOptimizationInfo(ParseContext pctx, } } + return extractSharedOptimizationInfo(pctx, optimizerCache, equalOp1, equalOp2, currentOp1, currentOp2, + retainableOps, discardableOps, discardableInputOps, false); + } + + private static SharedResult extractSharedOptimizationInfo(ParseContext pctx, + SharedWorkOptimizerCache optimizerCache, + Operator retainableOpEqualParent, + Operator discardableOpEqualParent, + Operator retainableOp, + Operator discardableOp) throws SemanticException { + return extractSharedOptimizationInfo(pctx, optimizerCache, retainableOpEqualParent, discardableOpEqualParent, + retainableOp, discardableOp, new LinkedHashSet<>(), new LinkedHashSet<>(), new HashSet<>(), true); + } + + private static SharedResult extractSharedOptimizationInfo(ParseContext pctx, + SharedWorkOptimizerCache optimizerCache, + Operator retainableOpEqualParent, + Operator discardableOpEqualParent, + Operator retainableOp, + Operator discardableOp, + LinkedHashSet> retainableOps, + LinkedHashSet> discardableOps, + Set> discardableInputOps, + boolean removeInputBranch) throws SemanticException { + Operator equalOp1 = retainableOpEqualParent; + Operator equalOp2 = discardableOpEqualParent; + Operator currentOp1 = retainableOp; + Operator currentOp2 = discardableOp; + long dataSize = 0l; + long maxDataSize = 0l; // Try to merge rest of operators while (!(currentOp1 instanceof ReduceSinkOperator)) { // Check whether current operators are equal @@ -563,7 +802,7 @@ private static SharedResult extractSharedOptimizationInfo(ParseContext pctx, for (; idx < currentOp1.getParentOperators().size(); idx++) { Operator parentOp1 = currentOp1.getParentOperators().get(idx); Operator parentOp2 = currentOp2.getParentOperators().get(idx); - if (parentOp1 == equalOp1 && parentOp2 == equalOp2) { + if (parentOp1 == equalOp1 && parentOp2 == equalOp2 && !removeInputBranch) { continue; } if ((parentOp1 == equalOp1 && parentOp2 != equalOp2) || @@ -741,7 +980,6 @@ private static boolean compareAndGatherOps(ParseContext pctx, Operator op1, O return true; } - @SuppressWarnings({ "rawtypes", "unchecked" }) private static boolean compareOperator(ParseContext pctx, Operator op1, Operator op2) throws SemanticException { if (!op1.getClass().getName().equals(op2.getClass().getName())) { @@ -809,21 +1047,21 @@ private static boolean validPreConditions(ParseContext pctx, SharedWorkOptimizer return false; } - TableScanOperator tsOp1 = (TableScanOperator) sr.retainableOps.get(0); - TableScanOperator tsOp2 = (TableScanOperator) sr.discardableOps.get(0); + Operator op1 = sr.retainableOps.get(0); + Operator op2 = sr.discardableOps.get(0); - // 1) The set of operators in the works of the TS operators need to meet + // 1) The set of operators in the works that we are merging need to meet // some requirements. In particular: - // 1.1. None of the works that contain the TS operators can contain a Union + // 1.1. None of the works that we are merging can contain a Union // operator. This is not supported yet as we might end up with cycles in // the Tez DAG. // 1.2. There cannot be more than one DummyStore operator in the new resulting - // work when the TS operators are merged. This is due to an assumption in + // work when the operators are merged. This is due to an assumption in // MergeJoinProc that needs to be further explored. // If any of these conditions are not met, we cannot merge. // TODO: Extend rule so it can be applied for these cases. - final Set> workOps1 = findWorkOperators(optimizerCache, tsOp1); - final Set> workOps2 = findWorkOperators(optimizerCache, tsOp2); + final Set> workOps1 = findWorkOperators(optimizerCache, op1); + final Set> workOps2 = findWorkOperators(optimizerCache, op2); boolean foundDummyStoreOp = false; for (Operator op : workOps1) { if (op instanceof UnionOperator) { @@ -853,8 +1091,8 @@ private static boolean validPreConditions(ParseContext pctx, SharedWorkOptimizer // If we do, we cannot merge. The reason is that Tez currently does // not support parallel edges, i.e., multiple edges from same work x // into same work y. - final Set> outputWorksOps1 = findChildWorkOperators(pctx, optimizerCache, tsOp1); - final Set> outputWorksOps2 = findChildWorkOperators(pctx, optimizerCache, tsOp2); + final Set> outputWorksOps1 = findChildWorkOperators(pctx, optimizerCache, op1); + final Set> outputWorksOps2 = findChildWorkOperators(pctx, optimizerCache, op2); if (!Collections.disjoint(outputWorksOps1, outputWorksOps2)) { // We cannot merge return false; @@ -866,10 +1104,19 @@ private static boolean validPreConditions(ParseContext pctx, SharedWorkOptimizer // Work2 Work3 Work2 // // If we do, we cannot merge. The reason is the same as above, currently - // Tez currently does not support parallel edges. - final Set> inputWorksOps1 = findParentWorkOperators(pctx, optimizerCache, tsOp1); + // Tez does not support parallel edges. + // + // In the check, we exclude the inputs to the root operator that we are trying + // to merge (only useful for extended merging as TS do not have inputs). + final Set> excludeOps1 = sr.retainableOps.get(0).getNumParent() > 0 ? + ImmutableSet.copyOf(sr.retainableOps.get(0).getParentOperators()) : ImmutableSet.of(); + final Set> inputWorksOps1 = + findParentWorkOperators(pctx, optimizerCache, op1, excludeOps1); + final Set> excludeOps2 = sr.discardableOps.get(0).getNumParent() > 0 ? + Sets.union(ImmutableSet.copyOf(sr.discardableOps.get(0).getParentOperators()), sr.discardableInputOps) : + sr.discardableInputOps; final Set> inputWorksOps2 = - findParentWorkOperators(pctx, optimizerCache, tsOp2, sr.discardableInputOps); + findParentWorkOperators(pctx, optimizerCache, op2, excludeOps2); if (!Collections.disjoint(inputWorksOps1, inputWorksOps2)) { // We cannot merge return false; @@ -885,9 +1132,9 @@ private static boolean validPreConditions(ParseContext pctx, SharedWorkOptimizer // // If we do, we cannot merge, as we would end up with a cycle in the DAG. final Set> descendantWorksOps1 = - findDescendantWorkOperators(pctx, optimizerCache, tsOp1, sr.discardableInputOps); + findDescendantWorkOperators(pctx, optimizerCache, op1, sr.discardableInputOps); final Set> descendantWorksOps2 = - findDescendantWorkOperators(pctx, optimizerCache, tsOp2, sr.discardableInputOps); + findDescendantWorkOperators(pctx, optimizerCache, op2, sr.discardableInputOps); if (!Collections.disjoint(descendantWorksOps1, workOps2) || !Collections.disjoint(workOps1, descendantWorksOps2)) { return false; @@ -1120,6 +1367,11 @@ private SharedResult(Collection> retainableOps, Collection opToRemove, Operator replacementOp) { } } } + + @Override + public String toString() { + return "SharedWorkOptimizerCache { \n" + operatorToWorkOperators.toString() + "\n };"; + } } } diff --git a/ql/src/test/queries/clientpositive/sharedworkext.q b/ql/src/test/queries/clientpositive/sharedworkext.q new file mode 100644 index 0000000000..b1801eabea --- /dev/null +++ b/ql/src/test/queries/clientpositive/sharedworkext.q @@ -0,0 +1,53 @@ +EXPLAIN +SELECT a.key FROM +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) a +JOIN +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) b +ON a.key = b.key; + +SELECT a.key FROM +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) a +JOIN +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) b +ON a.key = b.key; + +EXPLAIN +SELECT a.key FROM +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) a +) a +JOIN +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) b +) b +ON a.key = b.key; + +SELECT a.key FROM +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) a +) a +JOIN +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) b +) b +ON a.key = b.key; diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 8ab5e3ae6d..dfced42eb9 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -2363,10 +2363,9 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) -Reducer 7 <- Map 5 (CUSTOM_SIMPLE_EDGE) Stage-0 Fetch Operator @@ -2384,41 +2383,37 @@ Stage-0 predicate:((_col2 = 0) or (_col5 is null and _col1 is not null and (_col3 >= _col2))) Merge Join Operator [MERGEJOIN_37] (rows=26 width=141) Conds:RS_24.UDFToDouble(_col1)=RS_25._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5"] + <-Reducer 6 [SIMPLE_EDGE] llap + SHUFFLE [RS_25] + PartitionCols:_col0 + Select Operator [SEL_20] (rows=1 width=12) + Output:["_col0","_col1"] + Group By Operator [GBY_19] (rows=1 width=8) + Output:["_col0"],aggregations:["avg(VALUE._col0)"] + <-Map 5 [CUSTOM_SIMPLE_EDGE] llap + PARTITION_ONLY_SHUFFLE [RS_18] + Group By Operator [GBY_5] (rows=1 width=76) + Output:["_col0"],aggregations:["avg(p_size)"] + Filter Operator [FIL_33] (rows=8 width=4) + predicate:(p_size < 10) + TableScan [TS_2] (rows=26 width=4) + default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] <-Reducer 2 [SIMPLE_EDGE] llap SHUFFLE [RS_24] PartitionCols:UDFToDouble(_col1) Merge Join Operator [MERGEJOIN_36] (rows=26 width=141) Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] + <-Reducer 6 [CUSTOM_SIMPLE_EDGE] llap + SHUFFLE [RS_22] + Group By Operator [GBY_12] (rows=1 width=16) + Output:["_col0","_col1"],aggregations:["count()","count(_col0)"] + Please refer to the previous Group By Operator [GBY_19] <-Map 1 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_21] Select Operator [SEL_1] (rows=26 width=125) Output:["_col0","_col1"] TableScan [TS_0] (rows=26 width=125) default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_size"] - <-Reducer 6 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_22] - Group By Operator [GBY_12] (rows=1 width=16) - Output:["_col0","_col1"],aggregations:["count()","count(_col0)"] - Group By Operator [GBY_7] (rows=1 width=8) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] - Group By Operator [GBY_5] (rows=1 width=76) - Output:["_col0"],aggregations:["avg(p_size)"] - Filter Operator [FIL_33] (rows=8 width=4) - predicate:(p_size < 10) - TableScan [TS_2] (rows=26 width=4) - default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_25] - PartitionCols:_col0 - Select Operator [SEL_20] (rows=1 width=12) - Output:["_col0","_col1"] - Group By Operator [GBY_19] (rows=1 width=8) - Output:["_col0"],aggregations:["avg(VALUE._col0)"] - <-Map 5 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_18] - Please refer to the previous Group By Operator [GBY_5] PREHOOK: query: explain select b.p_mfgr, min(p_retailprice) from part b diff --git a/ql/src/test/results/clientpositive/llap/sharedworkext.q.out b/ql/src/test/results/clientpositive/llap/sharedworkext.q.out new file mode 100644 index 0000000000..e56b1cebc4 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/sharedworkext.q.out @@ -0,0 +1,1076 @@ +PREHOOK: query: EXPLAIN +SELECT a.key FROM +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) a +JOIN +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) b +ON a.key = b.key +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT a.key FROM +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) a +JOIN +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) b +ON a.key = b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 5 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: a1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 809 Data size: 144002 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col2 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 528 Data size: 45936 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 528 Data size: 45936 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT a.key FROM +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) a +JOIN +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) b +ON a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT a.key FROM +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) a +JOIN +( + SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value +) b +ON a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 +10 +100 +103 +104 +105 +11 +111 +113 +114 +116 +118 +119 +12 +120 +125 +126 +128 +129 +131 +133 +134 +136 +137 +138 +143 +145 +146 +149 +15 +150 +152 +153 +155 +156 +157 +158 +160 +162 +163 +164 +165 +166 +167 +168 +169 +17 +170 +172 +174 +175 +176 +177 +178 +179 +18 +180 +181 +183 +186 +187 +189 +19 +190 +191 +192 +193 +194 +195 +196 +197 +199 +2 +20 +200 +201 +202 +203 +205 +207 +208 +209 +213 +214 +216 +217 +218 +219 +221 +222 +223 +224 +226 +228 +229 +230 +233 +235 +237 +238 +239 +24 +241 +242 +244 +247 +248 +249 +252 +255 +256 +257 +258 +26 +260 +262 +263 +265 +266 +27 +272 +273 +274 +275 +277 +278 +28 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +291 +292 +296 +298 +30 +302 +305 +306 +307 +308 +309 +310 +311 +315 +316 +317 +318 +321 +322 +323 +325 +327 +33 +331 +332 +333 +335 +336 +338 +339 +34 +341 +342 +344 +345 +348 +35 +351 +353 +356 +360 +362 +364 +365 +366 +367 +368 +369 +37 +373 +374 +375 +377 +378 +379 +382 +384 +386 +389 +392 +393 +394 +395 +396 +397 +399 +4 +400 +401 +402 +403 +404 +406 +407 +409 +41 +411 +413 +414 +417 +418 +419 +42 +421 +424 +427 +429 +43 +430 +431 +432 +435 +436 +437 +438 +439 +44 +443 +444 +446 +448 +449 +452 +453 +454 +455 +457 +458 +459 +460 +462 +463 +466 +467 +468 +469 +47 +470 +472 +475 +477 +478 +479 +480 +481 +482 +483 +484 +485 +487 +489 +490 +491 +492 +493 +494 +495 +496 +497 +498 +5 +51 +53 +54 +57 +58 +64 +65 +66 +67 +69 +70 +72 +74 +76 +77 +78 +8 +80 +82 +83 +84 +85 +86 +87 +9 +90 +92 +95 +96 +97 +98 +PREHOOK: query: EXPLAIN +SELECT a.key FROM +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) a +) a +JOIN +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) b +) b +ON a.key = b.key +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT a.key FROM +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) a +) a +JOIN +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) b +) b +ON a.key = b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 6 <- Reducer 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: a1 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 809 Data size: 144002 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col1 (type: string), _col2 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 0 (type: int), _col0 (type: string) + sort order: ++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: 0 (type: int), _col0 (type: string) + sort order: ++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 ASC NULLS FIRST + partition by: 0 + raw input shape: + window functions: + window function definition + alias: rank_window_0 + arguments: _col0 + name: rank + window function: GenericUDAFRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: rank_window_0 is not null (type: boolean) + Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: rank_window_0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 ASC NULLS FIRST + partition by: 0 + raw input shape: + window functions: + window function definition + alias: rank_window_0 + arguments: _col0 + name: rank + window function: GenericUDAFRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: rank_window_0 is not null (type: boolean) + Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: rank_window_0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT a.key FROM +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) a +) a +JOIN +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) b +) b +ON a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT a.key FROM +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) a +) a +JOIN +( + SELECT rank() OVER (ORDER BY key) AS key FROM + (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key) + GROUP BY a1.key, a2.value) b +) b +ON a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out index 975fd133b5..a10c00cfed 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -1663,13 +1663,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) - Reducer 4 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) + Reducer 7 <- Map 10 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) - Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 9 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1692,7 +1691,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 11 + Map 10 Map Operator Tree: TableScan alias: pp @@ -1714,11 +1713,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Map 5 @@ -1763,35 +1757,8 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -1874,6 +1841,16 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 7 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Group By Operator + keys: _col2 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE Reducer 8 Execution mode: llap Reduce Operator Tree: @@ -1892,24 +1869,24 @@ STAGE PLANS: Reducer 9 Execution mode: llap Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 _col1 (type: string), _col0 (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 14 Data size: 2744 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col2 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: _col0 is not null (type: boolean) Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: boolean) Stage: Stage-0 Fetch Operator @@ -2143,13 +2120,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) - Reducer 4 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 7 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 7 <- Map 10 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) - Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 9 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2172,7 +2148,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 11 + Map 10 Map Operator Tree: TableScan alias: pp @@ -2194,11 +2170,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Map 5 @@ -2243,35 +2214,8 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: string), _col0 (type: string) Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: string) - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: string) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: string) - Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: string) Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -2342,6 +2286,16 @@ STAGE PLANS: 1 _col0 (type: string), _col1 (type: string) outputColumnNames: _col1, _col2 Statistics: Num rows: 14 Data size: 2744 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col2 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(), count(_col2) keys: _col1 (type: string) @@ -2355,6 +2309,27 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: boolean) + Reducer 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2369,27 +2344,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 7 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 9 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 _col1 (type: string), _col0 (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 14 Data size: 2744 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col2 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -2855,12 +2809,11 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE) - Reducer 11 <- Map 9 (CUSTOM_SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 7 <- Map 6 (XPROD_EDGE), Reducer 10 (XPROD_EDGE) - Reducer 8 <- Reducer 11 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 8 <- Reducer 10 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2939,10 +2892,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: struct) Execution mode: llap LLAP IO: no inputs Reducer 10 @@ -2962,14 +2911,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: avg(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: double), true (type: boolean) outputColumnNames: _col0, _col1 diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 9cc126230d..60eaba187c 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -605,12 +605,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 6 (XPROD_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 7 (XPROD_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 4 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) + Reducer 7 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -639,12 +638,6 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -733,30 +726,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: avg(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(_col0) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) - outputColumnNames: _col2, _col5 - Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -794,7 +763,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: struct) - Reducer 8 + Reducer 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -812,6 +781,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: boolean) + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: avg(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(), count(_col0) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint), _col1 (type: bigint) Stage: Stage-0 Fetch Operator @@ -891,15 +877,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 5 (SIMPLE_EDGE) - Reducer 11 <- Reducer 10 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) - Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 8 <- Map 5 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Reducer 8 <- Reducer 6 (SIMPLE_EDGE) + Reducer 9 <- Reducer 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -933,93 +917,8 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) - outputColumnNames: _col2, _col5 - Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col2: string, _col5: int - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col5 ASC NULLS FIRST - partition by: _col2 - raw input shape: - window functions: - window function definition - alias: rank_window_0 - arguments: _col5 - name: rank - window function: GenericUDAFRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (rank_window_0 <= 2) (type: boolean) - Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: string), _col5 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: min(_col1) - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col1 (type: string), _col0 (type: int) - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -1131,33 +1030,6 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - keys: _col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) - outputColumnNames: _col2, _col5 - Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE PTF Operator Function definitions: Input definition @@ -1198,6 +1070,92 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 ASC NULLS FIRST + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: rank_window_0 + arguments: _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (rank_window_0 <= 2) (type: boolean) + Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: string), _col5 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: min(_col1) + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int) + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col0 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: string), _col0 (type: int) + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: boolean) Reducer 9 Execution mode: llap Reduce Operator Tree: @@ -1841,12 +1799,11 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 7 (XPROD_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 5 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Reducer 8 <- Reducer 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1885,12 +1842,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -1969,6 +1920,16 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Group By Operator + keys: _col1 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Reducer 7 Execution mode: llap Reduce Operator Tree: @@ -1982,29 +1943,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int) - outputColumnNames: _col1 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col1 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2069,11 +2007,10 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 6 (XPROD_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 4 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Reducer 7 <- Reducer 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2112,12 +2049,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -2184,6 +2115,16 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Group By Operator + keys: _col1 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Reducer 6 Execution mode: llap Reduce Operator Tree: @@ -2197,29 +2138,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int) - outputColumnNames: _col1 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col1 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2518,11 +2436,10 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 6 (XPROD_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 4 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Reducer 7 <- Reducer 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2561,12 +2478,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: double) - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -2633,6 +2544,16 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Reducer 6 Execution mode: llap Reduce Operator Tree: @@ -2646,29 +2567,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: floor(_col1) (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2716,14 +2614,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 11 <- Map 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Reducer 7 <- Reducer 5 (SIMPLE_EDGE) + Reducer 9 <- Map 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2758,15 +2654,9 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: int), _col0 (type: int) Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Reduce Output Operator - key expressions: _col2 (type: int), _col0 (type: int) - sort order: ++ - Map-reduce partition columns: _col2 (type: int), _col0 (type: int) - Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 9 + Map 8 Map Operator Tree: TableScan alias: part @@ -2781,39 +2671,8 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), (_col0 + 121150) (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), (_col0 + 121150) (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -2879,6 +2738,16 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 6 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) + Group By Operator + keys: _col1 (type: string), _col3 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE Reducer 6 Execution mode: llap Reduce Operator Tree: @@ -2895,27 +2764,6 @@ STAGE PLANS: Statistics: Num rows: 6 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col2 (type: int), _col0 (type: int) - 1 _col0 (type: int), (_col0 + 121150) (type: int) - outputColumnNames: _col1, _col3 - Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col1 (type: string), _col3 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2936,6 +2784,19 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), (_col0 + 121150) (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -3322,14 +3183,12 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 11 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (ONE_TO_ONE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (ONE_TO_ONE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Reducer 7 <- Reducer 5 (SIMPLE_EDGE) + Reducer 9 <- Map 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3364,15 +3223,9 @@ STAGE PLANS: Map-reduce partition columns: (_col1 + 1) (type: int) Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reduce Output Operator - key expressions: (_col1 + 1) (type: int) - sort order: + - Map-reduce partition columns: (_col1 + 1) (type: int) - Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 9 + Map 8 Map Operator Tree: TableScan alias: part @@ -3391,39 +3244,8 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -3451,60 +3273,22 @@ STAGE PLANS: 0 _col1 (type: string), _col2 (type: int) 1 _col0 (type: string), _col1 (type: int) outputColumnNames: _col0, _col1, _col4, _col5, _col8 - Statistics: Num rows: 26 Data size: 6046 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (not CASE WHEN ((_col4 = 0)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) (type: boolean) - Statistics: Num rows: 13 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 (_col1 + 1) (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 16 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(), count(_col0) - keys: _col2 (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 8 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0), count(VALUE._col1) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 8 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 7 + Statistics: Num rows: 26 Data size: 6046 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (not CASE WHEN ((_col4 = 0)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col1 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) (type: boolean) + Statistics: Num rows: 13 Data size: 3025 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -3525,7 +3309,19 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 8 + Group By Operator + aggregations: count(), count(_col0) + keys: _col2 (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 8 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 8 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Reducer 6 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3546,6 +3342,34 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: int) Statistics: Num rows: 8 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: boolean) + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0), count(VALUE._col1) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 8 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 8 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -4175,12 +3999,11 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 7 (XPROD_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 5 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Reducer 8 <- Reducer 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4219,12 +4042,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -4303,6 +4120,16 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Group By Operator + keys: _col1 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Reducer 7 Execution mode: llap Reduce Operator Tree: @@ -4316,29 +4143,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int) - outputColumnNames: _col1 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col1 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -4403,12 +4207,11 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 7 (XPROD_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 5 (SIMPLE_EDGE) - Reducer 9 <- Reducer 8 (SIMPLE_EDGE) + Reducer 8 <- Reducer 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4447,12 +4250,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int) Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -4535,6 +4332,16 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) + Group By Operator + keys: _col1 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE Reducer 7 Execution mode: llap Reduce Operator Tree: @@ -4548,29 +4355,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int) - outputColumnNames: _col1 - Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col1 (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5747,14 +5531,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) - Reducer 3 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) - Reducer 6 <- Map 11 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) - Reducer 8 <- Map 4 (SIMPLE_EDGE) - Reducer 9 <- Map 11 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 8 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 9 <- Reducer 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -5774,7 +5557,7 @@ STAGE PLANS: value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 11 + Map 10 Map Operator Tree: TableScan alias: t2 @@ -5812,34 +5595,8 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: char(100)) Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: char(100)) - sort order: + - Map-reduce partition columns: _col0 (type: char(100)) - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: char(100)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: _col0 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), _col1 (type: char(100)), true (type: boolean) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: char(100)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -5896,7 +5653,55 @@ STAGE PLANS: Map-reduce partition columns: UDFToDouble(_col0) (type: double) Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: char(100)) + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: char(100)) Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col1 (type: char(100)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: char(100)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: char(100)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: char(100)), true (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: char(100)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: boolean) + Reducer 8 Execution mode: llap Reduce Operator Tree: Merge Join Operator @@ -5919,7 +5724,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: char(100)) Statistics: Num rows: 1 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 7 + Reducer 9 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5934,41 +5739,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: char(100)) Statistics: Num rows: 1 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: char(100)) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: char(100)) - Reducer 9 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(_col0) (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int), _col1 (type: char(100)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: char(100)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100)) - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out index 64337ed331..ba5dbb98fe 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out @@ -946,12 +946,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 6 (XPROD_EDGE), Reducer 8 (XPROD_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 5 (XPROD_EDGE), Reducer 7 (XPROD_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 7 <- Map 3 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (SIMPLE_EDGE) + Reducer 6 <- Reducer 4 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -979,11 +978,6 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: p_mfgr (type: string), p_size (type: int) - sort order: ++ - Map-reduce partition columns: p_mfgr (type: string) - Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 2 @@ -1046,46 +1040,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Select Operator - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 6 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (sq_count_check(_col0) <= 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 7 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) - outputColumnNames: _col2, _col5 - Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 PTF Operator Function definitions: Input definition @@ -1114,8 +1069,7 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - Reducer 8 + Reducer 5 Execution mode: llap Reduce Operator Tree: Select Operator @@ -1129,6 +1083,39 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Select Operator + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (sq_count_check(_col0) <= 1) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-0 Fetch Operator @@ -2770,18 +2757,16 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 11 (CONTAINS) - Reducer 12 <- Union 11 (SIMPLE_EDGE) - Reducer 14 <- Map 13 (CUSTOM_SIMPLE_EDGE) - Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Union 6 (CONTAINS) - Reducer 16 <- Map 13 (CUSTOM_SIMPLE_EDGE) - Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Union 11 (CONTAINS) - Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 12 (XPROD_EDGE), Reducer 8 (XPROD_EDGE) + Reducer 10 <- Union 9 (SIMPLE_EDGE) + Reducer 11 <- Reducer 10 (CUSTOM_SIMPLE_EDGE) + Reducer 13 <- Map 12 (CUSTOM_SIMPLE_EDGE) + Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 15 <- Reducer 13 (SIMPLE_EDGE), Union 9 (CONTAINS) + Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 11 (XPROD_EDGE), Reducer 7 (XPROD_EDGE) Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) Reducer 7 <- Union 6 (SIMPLE_EDGE) - Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 3 (CUSTOM_SIMPLE_EDGE) + Reducer 8 <- Reducer 4 (SIMPLE_EDGE), Union 9 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 @@ -2799,7 +2784,7 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 13 + Map 12 Map Operator Tree: TableScan alias: part @@ -2817,10 +2802,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs Map 3 @@ -2841,10 +2822,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) Execution mode: llap LLAP IO: no inputs Reducer 10 @@ -2856,39 +2833,41 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(_col1) - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reducer 12 + Select Operator + expressions: _col1 (type: bigint) + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col1 = 2) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 11 Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (_col1 = 2) (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (sq_count_check(_col0) <= 1) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: _col0 (type: bigint) - outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 14 + Reducer 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -2908,37 +2887,23 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 15 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: bigint) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(_col1) - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: bigint) sort order: + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 16 + Reducer 14 Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) + keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: count() + aggregations: count(_col1) keys: _col0 (type: bigint) mode: hash outputColumnNames: _col0, _col1 @@ -2949,7 +2914,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Reducer 17 + Reducer 15 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -3015,6 +2980,12 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) Reducer 5 Execution mode: llap Reduce Operator Tree: @@ -3045,50 +3016,28 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: bigint) - outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (_col1 = 2) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reducer 8 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: (sq_count_check(_col0) <= 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + predicate: (_col1 = 2) (type: boolean) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 9 + value expressions: _col0 (type: bigint) + Reducer 8 Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) + keys: KEY._col0 (type: bigint) mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - aggregations: count() + aggregations: count(_col1) keys: _col0 (type: bigint) mode: hash outputColumnNames: _col0, _col1 @@ -3099,10 +3048,10 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: bigint) - Union 11 - Vertex: Union 11 Union 6 Vertex: Union 6 + Union 9 + Vertex: Union 9 Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out b/ql/src/test/results/clientpositive/llap/subquery_select.q.out index b090cd099f..4d67a85e98 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out @@ -3083,11 +3083,10 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE) - Reducer 11 <- Map 9 (CUSTOM_SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 4 <- Reducer 10 (XPROD_EDGE), Reducer 3 (XPROD_EDGE) - Reducer 5 <- Reducer 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) Reducer 8 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -3155,10 +3154,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 10 @@ -3178,14 +3173,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Reducer 11 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), true (type: boolean) outputColumnNames: _col0, _col1 @@ -3503,7 +3490,6 @@ POSTHOOK: Input: default@part 45 false Warning: Shuffle Join MERGEJOIN[50][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product Warning: Shuffle Join MERGEJOIN[51][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[52][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 9' is a cross product PREHOOK: query: explain select p_size, (p_size IN (select (select max(p_size) from part) as sb from part order by sb limit 1)) = true from part @@ -3521,15 +3507,13 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 7 (XPROD_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 11 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) + Reducer 5 <- Map 9 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 8 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Reducer 9 <- Map 11 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) + Reducer 8 <- Reducer 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3558,22 +3542,9 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) - Select Operator - expressions: p_size (type: int) - outputColumnNames: p_size - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: max(p_size) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) Execution mode: llap LLAP IO: no inputs - Map 11 + Map 9 Map Operator Tree: TableScan alias: part @@ -3583,31 +3554,8 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), true (type: boolean) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: boolean) Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -3678,6 +3626,11 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Reducer 6 Execution mode: llap Reduce Operator Tree: @@ -3712,35 +3665,23 @@ STAGE PLANS: Reducer 8 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: max(VALUE._col0) - mode: mergepartial + Select Operator + expressions: KEY.reducesinkkey0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Reducer 9 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Left Outer Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col1 Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), true (type: boolean) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: boolean) Stage: Stage-0 Fetch Operator @@ -3750,7 +3691,6 @@ STAGE PLANS: Warning: Shuffle Join MERGEJOIN[50][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product Warning: Shuffle Join MERGEJOIN[51][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product -Warning: Shuffle Join MERGEJOIN[52][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 9' is a cross product PREHOOK: query: select p_size, (p_size IN (select (select max(p_size) from part) as sb from part order by sb limit 1)) = true from part diff --git a/ql/src/test/results/clientpositive/llap/subquery_views.q.out b/ql/src/test/results/clientpositive/llap/subquery_views.q.out index af695691a7..93913625ee 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -124,17 +124,14 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 13 (ONE_TO_ONE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 12 <- Map 11 (SIMPLE_EDGE) - Reducer 13 <- Map 11 (SIMPLE_EDGE) + Reducer 10 <- Map 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) - Reducer 6 <- Reducer 12 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 6 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 8 <- Map 1 (SIMPLE_EDGE) - Reducer 9 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -211,22 +208,9 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Filter Operator - predicate: ((value > 'val_11') and key is not null) (type: boolean) - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: key (type: string), value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Map 11 + Map 9 Map Operator Tree: TableScan alias: b @@ -243,44 +227,9 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Reducer 10 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - outputColumnNames: _col2, _col3, _col4 - Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: string), _col2 (type: string) - sort order: ++ - Map-reduce partition columns: _col4 (type: string), _col2 (type: string) - Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: boolean) - Reducer 12 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE - Reducer 13 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -387,6 +336,12 @@ STAGE PLANS: Map-reduce partition columns: _col4 (type: string), _col2 (type: string) Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col3 (type: boolean) + Reduce Output Operator + key expressions: _col4 (type: string), _col2 (type: string) + sort order: ++ + Map-reduce partition columns: _col4 (type: string), _col2 (type: string) + Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: boolean) Reducer 7 Execution mode: llap Reduce Operator Tree: @@ -432,24 +387,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: bigint), _col3 (type: bigint) - Reducer 9 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), true (type: boolean) - outputColumnNames: _col0, _col2, _col3 - Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: string), _col3 (type: boolean) Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out index 24c7ce7f76..e6075c79c0 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out @@ -1073,9 +1073,8 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 1 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 5 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1118,18 +1117,6 @@ STAGE PLANS: partitionColumnNums: [3] valueColumnNums: [] Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int) - sort order: +++ - Map-reduce partition columns: rand() (type: double) - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumnNums: [0, 1, 2] - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - partitionColumnNums: [4] - valueColumnNums: [] - Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -1187,6 +1174,18 @@ STAGE PLANS: partitionColumnNums: [0, 1] valueColumnNums: [] Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumnNums: [0, 1, 2] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumnNums: [0, 1] + valueColumnNums: [] + Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: @@ -1253,46 +1252,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 5 - Execution mode: vectorized, llap - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - reduceColumnNullOrder: aaa - reduceColumnSortOrder: +++ - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 3 - dataColumns: KEY._col0:int, KEY._col1:int, KEY._col2:int - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Reduce Operator Tree: - Group By Operator - Group By Vectorization: - className: VectorGroupByOperator - groupByMode: PARTIALS - keyExpressions: col 0:int, col 1:int, col 2:int - native: false - vectorProcessingMode: STREAMING - projectedOutputColumnNums: [] - keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: int) - mode: partials - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumnNums: [0, 1, 2] - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - partitionColumnNums: [0, 1] - valueColumnNums: [] - Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE - Reducer 6 Execution mode: vectorized, llap Reduce Vectorization: enabled: true @@ -1442,9 +1401,8 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 1 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 5 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1487,18 +1445,6 @@ STAGE PLANS: partitionColumnNums: [3] valueColumnNums: [] Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int) - sort order: +++ - Map-reduce partition columns: rand() (type: double) - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumnNums: [0, 1, 2] - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - partitionColumnNums: [4] - valueColumnNums: [] - Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -1556,6 +1502,18 @@ STAGE PLANS: partitionColumnNums: [0, 1] valueColumnNums: [] Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumnNums: [0, 1, 2] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumnNums: [0, 1] + valueColumnNums: [] + Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: @@ -1622,46 +1580,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 5 - Execution mode: vectorized, llap - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - reduceColumnNullOrder: aaa - reduceColumnSortOrder: +++ - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 3 - dataColumns: KEY._col0:int, KEY._col1:int, KEY._col2:int - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Reduce Operator Tree: - Group By Operator - Group By Vectorization: - className: VectorGroupByOperator - groupByMode: PARTIALS - keyExpressions: col 0:int, col 1:int, col 2:int - native: false - vectorProcessingMode: STREAMING - projectedOutputColumnNums: [] - keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: int) - mode: partials - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumnNums: [0, 1, 2] - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - partitionColumnNums: [0, 1] - valueColumnNums: [] - Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE - Reducer 6 Execution mode: vectorized, llap Reduce Vectorization: enabled: true diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out index 957bc22436..6a5e679795 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out @@ -576,9 +576,8 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 5 <- Map 1 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 5 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -622,18 +621,6 @@ STAGE PLANS: valueColumnNums: [2] Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Reduce Sink Vectorization: - className: VectorReduceSinkMultiKeyOperator - keyColumnNums: [0, 1] - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumnNums: [2] - Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -693,6 +680,18 @@ STAGE PLANS: valueColumnNums: [3] Statistics: Num rows: 8 Data size: 2944 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkMultiKeyOperator + keyColumnNums: [0, 1, 2] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumnNums: [3] + Statistics: Num rows: 8 Data size: 2944 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: bigint) Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: @@ -770,48 +769,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 5 - Execution mode: vectorized, llap - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - reduceColumnNullOrder: aa - reduceColumnSortOrder: ++ - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 3 - dataColumns: KEY._col0:string, KEY._col1:string, VALUE._col0:bigint - partitionColumnCount: 0 - scratchColumnTypeNames: [bigint] - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - Group By Vectorization: - aggregators: VectorUDAFCountMerge(col 2:bigint) -> bigint - className: VectorGroupByOperator - groupByMode: PARTIALS - keyExpressions: col 0:string, col 1:string, ConstantVectorExpression(val 0) -> 3:int - native: false - vectorProcessingMode: STREAMING - projectedOutputColumnNums: [0] - keys: KEY._col0 (type: string), KEY._col1 (type: string), 0 (type: int) - mode: partials - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 2944 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int) - Reduce Sink Vectorization: - className: VectorReduceSinkMultiKeyOperator - keyColumnNums: [0, 1, 2] - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumnNums: [3] - Statistics: Num rows: 8 Data size: 2944 Basic stats: COMPLETE Column stats: NONE - value expressions: _col3 (type: bigint) - Reducer 6 Execution mode: vectorized, llap Reduce Vectorization: enabled: true diff --git a/ql/src/test/results/clientpositive/perf/tez/query14.q.out b/ql/src/test/results/clientpositive/perf/tez/query14.q.out index b2a45f155a..c095548aad 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query14.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query14.q.out @@ -212,12 +212,12 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 11 <- Union 10 (CUSTOM_SIMPLE_EDGE) Reducer 12 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 27 (CUSTOM_SIMPLE_EDGE), Reducer 52 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 13 <- Map 1 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 14 (CONTAINS) +Reducer 13 <- Map 1 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 14 (CONTAINS) Reducer 15 <- Union 14 (CUSTOM_SIMPLE_EDGE) Reducer 16 <- Reducer 15 (CUSTOM_SIMPLE_EDGE), Reducer 30 (CUSTOM_SIMPLE_EDGE), Reducer 55 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 19 <- Map 17 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE), Union 10 (CONTAINS) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 3 (CONTAINS) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 20 <- Map 17 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE), Union 14 (CONTAINS) Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 43 (SIMPLE_EDGE), Union 23 (CONTAINS) Reducer 24 <- Union 23 (CUSTOM_SIMPLE_EDGE) @@ -232,58 +232,49 @@ Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 44 (SIMPLE_EDGE), Union 23 (CONTAINS) Reducer 37 <- Map 35 (SIMPLE_EDGE), Map 44 (SIMPLE_EDGE), Union 26 (CONTAINS) Reducer 38 <- Map 35 (SIMPLE_EDGE), Map 44 (SIMPLE_EDGE), Union 29 (CONTAINS) Reducer 4 <- Union 3 (CUSTOM_SIMPLE_EDGE) -Reducer 40 <- Map 39 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 23 (CONTAINS) -Reducer 41 <- Map 39 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 26 (CONTAINS) -Reducer 42 <- Map 39 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 29 (CONTAINS) +Reducer 40 <- Map 39 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 23 (CONTAINS) +Reducer 41 <- Map 39 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 26 (CONTAINS) +Reducer 42 <- Map 39 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 29 (CONTAINS) Reducer 46 <- Map 45 (SIMPLE_EDGE), Map 49 (SIMPLE_EDGE) Reducer 47 <- Map 56 (SIMPLE_EDGE), Reducer 46 (SIMPLE_EDGE), Reducer 58 (ONE_TO_ONE_EDGE) Reducer 48 <- Reducer 47 (SIMPLE_EDGE) Reducer 5 <- Reducer 24 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE), Reducer 48 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS) -Reducer 50 <- Map 100 (SIMPLE_EDGE), Map 49 (SIMPLE_EDGE) +Reducer 50 <- Map 49 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE) Reducer 51 <- Map 56 (SIMPLE_EDGE), Reducer 50 (SIMPLE_EDGE), Reducer 68 (ONE_TO_ONE_EDGE) Reducer 52 <- Reducer 51 (SIMPLE_EDGE) -Reducer 53 <- Map 101 (SIMPLE_EDGE), Map 49 (SIMPLE_EDGE) -Reducer 54 <- Map 56 (SIMPLE_EDGE), Reducer 53 (SIMPLE_EDGE), Reducer 78 (ONE_TO_ONE_EDGE) +Reducer 53 <- Map 49 (SIMPLE_EDGE), Map 92 (SIMPLE_EDGE) +Reducer 54 <- Map 56 (SIMPLE_EDGE), Reducer 53 (SIMPLE_EDGE), Reducer 77 (ONE_TO_ONE_EDGE) Reducer 55 <- Reducer 54 (SIMPLE_EDGE) Reducer 57 <- Map 56 (SIMPLE_EDGE), Reducer 62 (ONE_TO_ONE_EDGE) Reducer 58 <- Reducer 57 (SIMPLE_EDGE) -Reducer 59 <- Map 56 (SIMPLE_EDGE), Reducer 88 (SIMPLE_EDGE) +Reducer 59 <- Map 56 (SIMPLE_EDGE), Reducer 85 (SIMPLE_EDGE) Reducer 60 <- Reducer 59 (SIMPLE_EDGE), Union 61 (CONTAINS) Reducer 62 <- Union 61 (SIMPLE_EDGE) -Reducer 63 <- Map 56 (SIMPLE_EDGE), Reducer 92 (SIMPLE_EDGE) +Reducer 63 <- Map 56 (SIMPLE_EDGE), Reducer 87 (SIMPLE_EDGE) Reducer 64 <- Reducer 63 (SIMPLE_EDGE), Union 61 (CONTAINS) -Reducer 65 <- Map 56 (SIMPLE_EDGE), Reducer 93 (SIMPLE_EDGE) +Reducer 65 <- Map 56 (SIMPLE_EDGE), Reducer 88 (SIMPLE_EDGE) Reducer 66 <- Reducer 65 (SIMPLE_EDGE), Union 61 (CONTAINS) Reducer 67 <- Map 56 (SIMPLE_EDGE), Reducer 72 (ONE_TO_ONE_EDGE) Reducer 68 <- Reducer 67 (SIMPLE_EDGE) -Reducer 69 <- Map 56 (SIMPLE_EDGE), Reducer 89 (SIMPLE_EDGE) +Reducer 69 <- Map 56 (SIMPLE_EDGE), Reducer 85 (SIMPLE_EDGE) Reducer 7 <- Union 6 (SIMPLE_EDGE) Reducer 70 <- Reducer 69 (SIMPLE_EDGE), Union 71 (CONTAINS) Reducer 72 <- Union 71 (SIMPLE_EDGE) -Reducer 73 <- Map 56 (SIMPLE_EDGE), Reducer 94 (SIMPLE_EDGE) -Reducer 74 <- Reducer 73 (SIMPLE_EDGE), Union 71 (CONTAINS) -Reducer 75 <- Map 56 (SIMPLE_EDGE), Reducer 95 (SIMPLE_EDGE) -Reducer 76 <- Reducer 75 (SIMPLE_EDGE), Union 71 (CONTAINS) -Reducer 77 <- Map 56 (SIMPLE_EDGE), Reducer 82 (ONE_TO_ONE_EDGE) -Reducer 78 <- Reducer 77 (SIMPLE_EDGE) -Reducer 79 <- Map 56 (SIMPLE_EDGE), Reducer 90 (SIMPLE_EDGE) +Reducer 73 <- Reducer 69 (SIMPLE_EDGE), Union 74 (CONTAINS) +Reducer 75 <- Union 74 (SIMPLE_EDGE) +Reducer 76 <- Map 56 (SIMPLE_EDGE), Reducer 75 (ONE_TO_ONE_EDGE) +Reducer 77 <- Reducer 76 (SIMPLE_EDGE) +Reducer 78 <- Map 56 (SIMPLE_EDGE), Reducer 87 (SIMPLE_EDGE) +Reducer 79 <- Reducer 78 (SIMPLE_EDGE), Union 71 (CONTAINS) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) -Reducer 80 <- Reducer 79 (SIMPLE_EDGE), Union 81 (CONTAINS) -Reducer 82 <- Union 81 (SIMPLE_EDGE) -Reducer 83 <- Map 56 (SIMPLE_EDGE), Reducer 96 (SIMPLE_EDGE) -Reducer 84 <- Reducer 83 (SIMPLE_EDGE), Union 81 (CONTAINS) -Reducer 85 <- Map 56 (SIMPLE_EDGE), Reducer 97 (SIMPLE_EDGE) -Reducer 86 <- Reducer 85 (SIMPLE_EDGE), Union 81 (CONTAINS) -Reducer 88 <- Map 87 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE) -Reducer 89 <- Map 87 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE) -Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 10 (CONTAINS) -Reducer 90 <- Map 87 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE) -Reducer 92 <- Map 91 (SIMPLE_EDGE), Map 98 (SIMPLE_EDGE) -Reducer 93 <- Map 91 (SIMPLE_EDGE), Map 99 (SIMPLE_EDGE) -Reducer 94 <- Map 91 (SIMPLE_EDGE), Map 98 (SIMPLE_EDGE) -Reducer 95 <- Map 91 (SIMPLE_EDGE), Map 99 (SIMPLE_EDGE) -Reducer 96 <- Map 91 (SIMPLE_EDGE), Map 98 (SIMPLE_EDGE) -Reducer 97 <- Map 91 (SIMPLE_EDGE), Map 99 (SIMPLE_EDGE) +Reducer 80 <- Reducer 78 (SIMPLE_EDGE), Union 74 (CONTAINS) +Reducer 81 <- Map 56 (SIMPLE_EDGE), Reducer 88 (SIMPLE_EDGE) +Reducer 82 <- Reducer 81 (SIMPLE_EDGE), Union 74 (CONTAINS) +Reducer 83 <- Reducer 81 (SIMPLE_EDGE), Union 71 (CONTAINS) +Reducer 85 <- Map 84 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE) +Reducer 87 <- Map 86 (SIMPLE_EDGE), Map 89 (SIMPLE_EDGE) +Reducer 88 <- Map 86 (SIMPLE_EDGE), Map 90 (SIMPLE_EDGE) +Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 10 (CONTAINS) Stage-0 Fetch Operator @@ -390,7 +381,7 @@ Stage-0 Output:["_col0"] Merge Join Operator [MERGEJOIN_860] (rows=633595212 width=88) Conds:RS_194._col0=RS_195._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] + <-Map 86 [SIMPLE_EDGE] SHUFFLE [RS_195] PartitionCols:_col0 Select Operator [SEL_98] (rows=8116 width=1119) @@ -469,7 +460,7 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_863] (rows=633595212 width=88) Conds:RS_238._col0=RS_239._col0(Inner),Output:["_col1","_col2"] - <-Map 91 [SIMPLE_EDGE] + <-Map 86 [SIMPLE_EDGE] SHUFFLE [RS_239] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_98] @@ -518,7 +509,7 @@ Stage-0 predicate:((d_moy = 11) and (d_year = 2000) and d_date_sk is not null) TableScan [TS_84] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 100 [SIMPLE_EDGE] + <-Map 91 [SIMPLE_EDGE] SHUFFLE [RS_357] PartitionCols:_col0 Select Operator [SEL_271] (rows=287989836 width=135) @@ -579,17 +570,17 @@ Stage-0 Filter Operator [FIL_813] (rows=462000 width=1436) predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null) Please refer to the previous TableScan [TS_90] - <-Reducer 89 [SIMPLE_EDGE] + <-Reducer 85 [SIMPLE_EDGE] SHUFFLE [RS_293] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_867] (rows=633595212 width=88) - Conds:RS_290._col0=RS_291._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] - SHUFFLE [RS_291] + Merge Join Operator [MERGEJOIN_852] (rows=633595212 width=88) + Conds:RS_102._col0=RS_103._col0(Inner),Output:["_col1"] + <-Map 86 [SIMPLE_EDGE] + SHUFFLE [RS_103] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_98] - <-Map 87 [SIMPLE_EDGE] - SHUFFLE [RS_290] + <-Map 84 [SIMPLE_EDGE] + SHUFFLE [RS_102] PartitionCols:_col0 Select Operator [SEL_95] (rows=575995635 width=88) Output:["_col0","_col1"] @@ -597,14 +588,14 @@ Stage-0 predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) TableScan [TS_93] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Reducer 74 [CONTAINS] + <-Reducer 79 [CONTAINS] Reduce Output Operator [RS_345] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_344] (rows=609832849 width=108) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 Group By Operator [GBY_319] (rows=174233858 width=135) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 73 [SIMPLE_EDGE] + <-Reducer 78 [SIMPLE_EDGE] SHUFFLE [RS_318] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_317] (rows=348467716 width=135) @@ -619,17 +610,17 @@ Stage-0 Filter Operator [FIL_816] (rows=462000 width=1436) predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null) Please refer to the previous TableScan [TS_90] - <-Reducer 94 [SIMPLE_EDGE] + <-Reducer 87 [SIMPLE_EDGE] SHUFFLE [RS_313] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_869] (rows=316788826 width=135) - Conds:RS_310._col0=RS_311._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] - SHUFFLE [RS_311] + Merge Join Operator [MERGEJOIN_854] (rows=316788826 width=135) + Conds:RS_122._col0=RS_123._col0(Inner),Output:["_col1"] + <-Map 86 [SIMPLE_EDGE] + SHUFFLE [RS_123] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_98] - <-Map 98 [SIMPLE_EDGE] - SHUFFLE [RS_310] + <-Map 89 [SIMPLE_EDGE] + SHUFFLE [RS_122] PartitionCols:_col0 Select Operator [SEL_115] (rows=287989836 width=135) Output:["_col0","_col1"] @@ -637,39 +628,39 @@ Stage-0 predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) TableScan [TS_113] (rows=287989836 width=135) default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk"] - <-Reducer 76 [CONTAINS] + <-Reducer 83 [CONTAINS] Reduce Output Operator [RS_345] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_344] (rows=609832849 width=108) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 Group By Operator [GBY_340] (rows=87121617 width=135) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 75 [SIMPLE_EDGE] + <-Reducer 81 [SIMPLE_EDGE] SHUFFLE [RS_339] PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_338] (rows=174243235 width=135) + Group By Operator [GBY_527] (rows=174243235 width=135) Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7 - Merge Join Operator [MERGEJOIN_872] (rows=174243235 width=135) - Conds:RS_334._col1=RS_335._col0(Inner),Output:["_col5","_col6","_col7"] + Merge Join Operator [MERGEJOIN_887] (rows=174243235 width=135) + Conds:RS_523._col1=RS_524._col0(Inner),Output:["_col5","_col6","_col7"] <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_335] + SHUFFLE [RS_524] PartitionCols:_col0 - Select Operator [SEL_330] (rows=462000 width=1436) + Select Operator [SEL_519] (rows=462000 width=1436) Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_819] (rows=462000 width=1436) + Filter Operator [FIL_844] (rows=462000 width=1436) predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null) Please refer to the previous TableScan [TS_90] - <-Reducer 95 [SIMPLE_EDGE] - SHUFFLE [RS_334] + <-Reducer 88 [SIMPLE_EDGE] + SHUFFLE [RS_523] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_871] (rows=158402938 width=135) - Conds:RS_331._col0=RS_332._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] - SHUFFLE [RS_332] + Merge Join Operator [MERGEJOIN_886] (rows=158402938 width=135) + Conds:RS_520._col0=RS_521._col0(Inner),Output:["_col1"] + <-Map 86 [SIMPLE_EDGE] + SHUFFLE [RS_521] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_98] - <-Map 99 [SIMPLE_EDGE] - SHUFFLE [RS_331] + <-Map 90 [SIMPLE_EDGE] + SHUFFLE [RS_520] PartitionCols:_col0 Select Operator [SEL_136] (rows=144002668 width=135) Output:["_col0","_col1"] @@ -709,7 +700,7 @@ Stage-0 Output:["_col0"] Merge Join Operator [MERGEJOIN_875] (rows=633595212 width=88) Conds:RS_383._col0=RS_384._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] + <-Map 86 [SIMPLE_EDGE] SHUFFLE [RS_384] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_98] @@ -804,7 +795,7 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_878] (rows=633595212 width=88) Conds:RS_427._col0=RS_428._col0(Inner),Output:["_col1","_col2"] - <-Map 91 [SIMPLE_EDGE] + <-Map 86 [SIMPLE_EDGE] SHUFFLE [RS_428] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_98] @@ -842,7 +833,7 @@ Stage-0 SHUFFLE [RS_547] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_86] - <-Map 101 [SIMPLE_EDGE] + <-Map 92 [SIMPLE_EDGE] SHUFFLE [RS_546] PartitionCols:_col0 Select Operator [SEL_460] (rows=144002668 width=135) @@ -851,12 +842,12 @@ Stage-0 predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) TableScan [TS_458] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_quantity","ws_list_price"] - <-Reducer 78 [ONE_TO_ONE_EDGE] + <-Reducer 77 [ONE_TO_ONE_EDGE] FORWARD [RS_551] PartitionCols:_col0 Group By Operator [GBY_544] (rows=254100 width=1436) Output:["_col0"],keys:KEY._col0 - <-Reducer 77 [SIMPLE_EDGE] + <-Reducer 76 [SIMPLE_EDGE] SHUFFLE [RS_543] PartitionCols:_col0 Group By Operator [GBY_542] (rows=508200 width=1436) @@ -871,7 +862,7 @@ Stage-0 Filter Operator [FIL_835] (rows=462000 width=1436) predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null) Please refer to the previous TableScan [TS_90] - <-Reducer 82 [ONE_TO_ONE_EDGE] + <-Reducer 75 [ONE_TO_ONE_EDGE] FORWARD [RS_539] PartitionCols:_col0, _col1, _col2 Select Operator [SEL_537] (rows=1 width=108) @@ -880,112 +871,40 @@ Stage-0 predicate:(_col3 = 3) Group By Operator [GBY_535] (rows=304916424 width=108) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Union 81 [SIMPLE_EDGE] - <-Reducer 80 [CONTAINS] + <-Union 74 [SIMPLE_EDGE] + <-Reducer 73 [CONTAINS] Reduce Output Operator [RS_534] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_533] (rows=609832849 width=108) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 Group By Operator [GBY_488] (rows=348477374 width=88) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 79 [SIMPLE_EDGE] + <-Reducer 69 [SIMPLE_EDGE] SHUFFLE [RS_487] PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_486] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7 - Merge Join Operator [MERGEJOIN_883] (rows=696954748 width=88) - Conds:RS_482._col1=RS_483._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_483] - PartitionCols:_col0 - Select Operator [SEL_478] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_838] (rows=462000 width=1436) - predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null) - Please refer to the previous TableScan [TS_90] - <-Reducer 90 [SIMPLE_EDGE] - SHUFFLE [RS_482] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_882] (rows=633595212 width=88) - Conds:RS_479._col0=RS_480._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] - SHUFFLE [RS_480] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_98] - <-Map 87 [SIMPLE_EDGE] - SHUFFLE [RS_479] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_95] - <-Reducer 84 [CONTAINS] + Please refer to the previous Group By Operator [GBY_297] + <-Reducer 80 [CONTAINS] Reduce Output Operator [RS_534] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_533] (rows=609832849 width=108) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 Group By Operator [GBY_508] (rows=174233858 width=135) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 83 [SIMPLE_EDGE] + <-Reducer 78 [SIMPLE_EDGE] SHUFFLE [RS_507] PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_506] (rows=348467716 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7 - Merge Join Operator [MERGEJOIN_885] (rows=348467716 width=135) - Conds:RS_502._col1=RS_503._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_503] - PartitionCols:_col0 - Select Operator [SEL_498] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_841] (rows=462000 width=1436) - predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null) - Please refer to the previous TableScan [TS_90] - <-Reducer 96 [SIMPLE_EDGE] - SHUFFLE [RS_502] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_884] (rows=316788826 width=135) - Conds:RS_499._col0=RS_500._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] - SHUFFLE [RS_500] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_98] - <-Map 98 [SIMPLE_EDGE] - SHUFFLE [RS_499] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_115] - <-Reducer 86 [CONTAINS] + Please refer to the previous Group By Operator [GBY_317] + <-Reducer 82 [CONTAINS] Reduce Output Operator [RS_534] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_533] (rows=609832849 width=108) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2 Group By Operator [GBY_529] (rows=87121617 width=135) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 85 [SIMPLE_EDGE] + <-Reducer 81 [SIMPLE_EDGE] SHUFFLE [RS_528] PartitionCols:_col0, _col1, _col2 - Group By Operator [GBY_527] (rows=174243235 width=135) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7 - Merge Join Operator [MERGEJOIN_887] (rows=174243235 width=135) - Conds:RS_523._col1=RS_524._col0(Inner),Output:["_col5","_col6","_col7"] - <-Map 56 [SIMPLE_EDGE] - SHUFFLE [RS_524] - PartitionCols:_col0 - Select Operator [SEL_519] (rows=462000 width=1436) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_844] (rows=462000 width=1436) - predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null) - Please refer to the previous TableScan [TS_90] - <-Reducer 97 [SIMPLE_EDGE] - SHUFFLE [RS_523] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_886] (rows=158402938 width=135) - Conds:RS_520._col0=RS_521._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] - SHUFFLE [RS_521] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_98] - <-Map 99 [SIMPLE_EDGE] - SHUFFLE [RS_520] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_136] + Please refer to the previous Group By Operator [GBY_527] <-Reducer 5 [CONTAINS] Reduce Output Operator [RS_568] PartitionCols:_col0, _col1, _col2, _col3, _col4 @@ -1048,7 +967,7 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_848] (rows=633595212 width=88) Conds:RS_50._col0=RS_51._col0(Inner),Output:["_col1","_col2"] - <-Map 91 [SIMPLE_EDGE] + <-Map 86 [SIMPLE_EDGE] SHUFFLE [RS_51] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_98] @@ -1095,7 +1014,7 @@ Stage-0 Output:["_col0"] Merge Join Operator [MERGEJOIN_845] (rows=633595212 width=88) Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] + <-Map 86 [SIMPLE_EDGE] SHUFFLE [RS_7] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_98] @@ -1208,19 +1127,10 @@ Stage-0 SHUFFLE [RS_106] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_92] - <-Reducer 88 [SIMPLE_EDGE] + <-Reducer 85 [SIMPLE_EDGE] SHUFFLE [RS_105] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_852] (rows=633595212 width=88) - Conds:RS_102._col0=RS_103._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_98] - <-Map 87 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_95] + Please refer to the previous Merge Join Operator [MERGEJOIN_852] <-Reducer 64 [CONTAINS] Reduce Output Operator [RS_157] PartitionCols:_col0, _col1, _col2 @@ -1239,19 +1149,10 @@ Stage-0 SHUFFLE [RS_126] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_92] - <-Reducer 92 [SIMPLE_EDGE] + <-Reducer 87 [SIMPLE_EDGE] SHUFFLE [RS_125] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_854] (rows=316788826 width=135) - Conds:RS_122._col0=RS_123._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] - SHUFFLE [RS_123] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_98] - <-Map 98 [SIMPLE_EDGE] - SHUFFLE [RS_122] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_115] + Please refer to the previous Merge Join Operator [MERGEJOIN_854] <-Reducer 66 [CONTAINS] Reduce Output Operator [RS_157] PartitionCols:_col0, _col1, _col2 @@ -1270,17 +1171,8 @@ Stage-0 SHUFFLE [RS_147] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_92] - <-Reducer 93 [SIMPLE_EDGE] + <-Reducer 88 [SIMPLE_EDGE] SHUFFLE [RS_146] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_856] (rows=158402938 width=135) - Conds:RS_143._col0=RS_144._col0(Inner),Output:["_col1"] - <-Map 91 [SIMPLE_EDGE] - SHUFFLE [RS_144] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_98] - <-Map 99 [SIMPLE_EDGE] - SHUFFLE [RS_143] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_136] + Please refer to the previous Merge Join Operator [MERGEJOIN_886] diff --git a/ql/src/test/results/clientpositive/perf/tez/query2.q.out b/ql/src/test/results/clientpositive/perf/tez/query2.q.out index 23f8249d27..e32f3baaf8 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query2.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query2.q.out @@ -120,17 +120,13 @@ Plan optimized by CBO. Vertex dependency in root stage Map 1 <- Union 2 (CONTAINS) -Map 14 <- Union 15 (CONTAINS) -Map 16 <- Union 15 (CONTAINS) -Map 8 <- Union 2 (CONTAINS) -Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 15 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 13 (SIMPLE_EDGE), Reducer 11 (ONE_TO_ONE_EDGE) -Reducer 3 <- Map 9 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) +Map 9 <- Union 2 (CONTAINS) +Reducer 3 <- Map 10 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) -Reducer 6 <- Reducer 12 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) +Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) +Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Reducer 8 <- Map 11 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Stage-0 Fetch Operator @@ -146,12 +142,12 @@ Stage-0 Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] Merge Join Operator [MERGEJOIN_95] (rows=287491028 width=135) Conds:RS_54._col0=RS_55.(_col0 - 53)(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"] - <-Reducer 12 [SIMPLE_EDGE] + <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_55] PartitionCols:(_col0 - 53) Merge Join Operator [MERGEJOIN_94] (rows=261355475 width=135) Conds:RS_50._col0=RS_51._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 13 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_51] PartitionCols:_col0 Select Operator [SEL_49] (rows=36524 width=1119) @@ -160,22 +156,22 @@ Stage-0 predicate:((d_year = 2002) and d_week_seq is not null) TableScan [TS_20] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_week_seq","d_year"] - <-Reducer 11 [ONE_TO_ONE_EDGE] + <-Reducer 4 [ONE_TO_ONE_EDGE] FORWARD [RS_50] PartitionCols:_col0 Group By Operator [GBY_45] (rows=237595882 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)"],keys:KEY._col0 - <-Reducer 10 [SIMPLE_EDGE] + <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_44] PartitionCols:_col0 - Group By Operator [GBY_43] (rows=475191764 width=135) + Group By Operator [GBY_16] (rows=475191764 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)","sum(_col4)","sum(_col5)","sum(_col6)","sum(_col7)"],keys:_col0 - Select Operator [SEL_41] (rows=475191764 width=135) + Select Operator [SEL_14] (rows=475191764 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_93] (rows=475191764 width=135) - Conds:Union 15._col0=RS_39._col0(Inner),Output:["_col1","_col3","_col4"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_39] + Merge Join Operator [MERGEJOIN_91] (rows=475191764 width=135) + Conds:Union 2._col0=RS_12._col0(Inner),Output:["_col1","_col3","_col4"] + <-Map 10 [SIMPLE_EDGE] + SHUFFLE [RS_12] PartitionCols:_col0 Select Operator [SEL_10] (rows=73049 width=1119) Output:["_col0","_col1","_col2"] @@ -183,31 +179,31 @@ Stage-0 predicate:(d_date_sk is not null and d_week_seq is not null) TableScan [TS_8] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_week_seq","d_day_name"] - <-Union 15 [SIMPLE_EDGE] - <-Map 14 [CONTAINS] - Reduce Output Operator [RS_38] + <-Union 2 [SIMPLE_EDGE] + <-Map 1 [CONTAINS] + Reduce Output Operator [RS_11] PartitionCols:_col0 - Select Operator [SEL_29] (rows=144002668 width=135) + Select Operator [SEL_2] (rows=144002668 width=135) Output:["_col0","_col1"] - Filter Operator [FIL_87] (rows=144002668 width=135) + Filter Operator [FIL_83] (rows=144002668 width=135) predicate:ws_sold_date_sk is not null - TableScan [TS_27] (rows=144002668 width=135) + TableScan [TS_0] (rows=144002668 width=135) Output:["ws_sold_date_sk","ws_ext_sales_price"] - <-Map 16 [CONTAINS] - Reduce Output Operator [RS_38] + <-Map 9 [CONTAINS] + Reduce Output Operator [RS_11] PartitionCols:_col0 - Select Operator [SEL_32] (rows=287989836 width=135) + Select Operator [SEL_5] (rows=287989836 width=135) Output:["_col0","_col1"] - Filter Operator [FIL_88] (rows=287989836 width=135) + Filter Operator [FIL_84] (rows=287989836 width=135) predicate:cs_sold_date_sk is not null - TableScan [TS_30] (rows=287989836 width=135) + TableScan [TS_3] (rows=287989836 width=135) Output:["cs_sold_date_sk","cs_ext_sales_price"] - <-Reducer 5 [ONE_TO_ONE_EDGE] + <-Reducer 8 [ONE_TO_ONE_EDGE] FORWARD [RS_54] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_92] (rows=261355475 width=135) Conds:RS_23._col0=RS_24._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 13 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_24] PartitionCols:_col0 Select Operator [SEL_22] (rows=36524 width=1119) @@ -218,38 +214,5 @@ Stage-0 <-Reducer 4 [ONE_TO_ONE_EDGE] FORWARD [RS_23] PartitionCols:_col0 - Group By Operator [GBY_18] (rows=237595882 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col0 - Group By Operator [GBY_16] (rows=475191764 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)","sum(_col4)","sum(_col5)","sum(_col6)","sum(_col7)"],keys:_col0 - Select Operator [SEL_14] (rows=475191764 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - Merge Join Operator [MERGEJOIN_91] (rows=475191764 width=135) - Conds:Union 2._col0=RS_12._col0(Inner),Output:["_col1","_col3","_col4"] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_10] - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_11] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=144002668 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_83] (rows=144002668 width=135) - predicate:ws_sold_date_sk is not null - TableScan [TS_0] (rows=144002668 width=135) - Output:["ws_sold_date_sk","ws_ext_sales_price"] - <-Map 8 [CONTAINS] - Reduce Output Operator [RS_11] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=287989836 width=135) - Output:["_col0","_col1"] - Filter Operator [FIL_84] (rows=287989836 width=135) - predicate:cs_sold_date_sk is not null - TableScan [TS_3] (rows=287989836 width=135) - Output:["cs_sold_date_sk","cs_ext_sales_price"] + Please refer to the previous Group By Operator [GBY_45] diff --git a/ql/src/test/results/clientpositive/perf/tez/query23.q.out b/ql/src/test/results/clientpositive/perf/tez/query23.q.out index 7112de61d9..2416768666 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query23.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query23.q.out @@ -1,5 +1,5 @@ -Warning: Shuffle Join MERGEJOIN[367][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 25' is a cross product -Warning: Shuffle Join MERGEJOIN[369][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 30' is a cross product +Warning: Shuffle Join MERGEJOIN[367][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 22' is a cross product +Warning: Shuffle Join MERGEJOIN[369][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 23' is a cross product PREHOOK: query: explain with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt @@ -105,41 +105,28 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 30 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE) -Reducer 13 <- Map 19 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) +Reducer 10 <- Reducer 23 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS) +Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE) +Reducer 13 <- Map 16 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 11 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE) -Reducer 16 <- Map 19 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE) +Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) +Reducer 19 <- Map 33 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 21 <- Map 20 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 22 <- Map 46 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE) -Reducer 24 <- Reducer 23 (CUSTOM_SIMPLE_EDGE) -Reducer 25 <- Reducer 24 (CUSTOM_SIMPLE_EDGE), Reducer 35 (CUSTOM_SIMPLE_EDGE), Reducer 43 (CUSTOM_SIMPLE_EDGE) -Reducer 26 <- Map 20 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE) -Reducer 27 <- Map 46 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 28 <- Reducer 27 (SIMPLE_EDGE) -Reducer 29 <- Reducer 28 (CUSTOM_SIMPLE_EDGE) +Reducer 20 <- Reducer 19 (SIMPLE_EDGE) +Reducer 21 <- Reducer 20 (CUSTOM_SIMPLE_EDGE) +Reducer 22 <- Reducer 21 (CUSTOM_SIMPLE_EDGE), Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 32 (CUSTOM_SIMPLE_EDGE) +Reducer 23 <- Reducer 21 (CUSTOM_SIMPLE_EDGE), Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 32 (CUSTOM_SIMPLE_EDGE) +Reducer 25 <- Map 24 (SIMPLE_EDGE), Map 29 (SIMPLE_EDGE) +Reducer 26 <- Map 33 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) +Reducer 27 <- Reducer 26 (SIMPLE_EDGE) +Reducer 28 <- Reducer 27 (CUSTOM_SIMPLE_EDGE) Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (CUSTOM_SIMPLE_EDGE), Reducer 39 (CUSTOM_SIMPLE_EDGE), Reducer 45 (CUSTOM_SIMPLE_EDGE) -Reducer 32 <- Map 31 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE) -Reducer 33 <- Map 46 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) -Reducer 34 <- Reducer 33 (SIMPLE_EDGE) -Reducer 35 <- Reducer 34 (CUSTOM_SIMPLE_EDGE) -Reducer 36 <- Map 31 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE) -Reducer 37 <- Map 46 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 38 <- Reducer 37 (SIMPLE_EDGE) -Reducer 39 <- Reducer 38 (CUSTOM_SIMPLE_EDGE) -Reducer 4 <- Reducer 25 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 42 <- Map 41 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE) -Reducer 43 <- Reducer 42 (SIMPLE_EDGE) -Reducer 44 <- Map 41 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE) -Reducer 45 <- Reducer 44 (SIMPLE_EDGE) +Reducer 31 <- Map 30 (SIMPLE_EDGE), Map 33 (SIMPLE_EDGE) +Reducer 32 <- Reducer 31 (SIMPLE_EDGE) +Reducer 4 <- Reducer 22 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (CUSTOM_SIMPLE_EDGE) -Reducer 8 <- Map 47 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 17 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 8 <- Map 34 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) +Reducer 9 <- Reducer 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Stage-0 Fetch Operator @@ -160,7 +147,7 @@ Stage-0 Output:["_col0"] Merge Join Operator [MERGEJOIN_370] (rows=191667562 width=135) Conds:RS_234._col2=RS_235._col0(Inner),Output:["_col3","_col4"] - <-Reducer 30 [SIMPLE_EDGE] + <-Reducer 23 [SIMPLE_EDGE] SHUFFLE [RS_235] PartitionCols:_col0 Select Operator [SEL_227] (rows=105599202 width=321) @@ -169,22 +156,22 @@ Stage-0 predicate:(_col3 > (0.95 * _col1)) Merge Join Operator [MERGEJOIN_369] (rows=316797606 width=321) Conds:(Inner),(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 29 [CUSTOM_SIMPLE_EDGE] + <-Reducer 21 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_222] - Select Operator [SEL_180] (rows=1 width=8) - Filter Operator [FIL_179] (rows=1 width=8) + Select Operator [SEL_61] (rows=1 width=8) + Filter Operator [FIL_60] (rows=1 width=8) predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_177] (rows=1 width=8) + Group By Operator [GBY_58] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 28 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_176] - Group By Operator [GBY_175] (rows=1 width=8) + <-Reducer 20 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_57] + Group By Operator [GBY_56] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_172] (rows=348477374 width=88) - Group By Operator [GBY_171] (rows=348477374 width=88) + Select Operator [SEL_53] (rows=348477374 width=88) + Group By Operator [GBY_52] (rows=348477374 width=88) Output:["_col0"],keys:KEY._col0 - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_170] + <-Reducer 19 [SIMPLE_EDGE] + SHUFFLE [RS_51] PartitionCols:_col0 Group By Operator [GBY_169] (rows=696954748 width=88) Output:["_col0"],keys:_col0 @@ -192,7 +179,7 @@ Stage-0 Output:["_col0"] Merge Join Operator [MERGEJOIN_361] (rows=696954748 width=88) Conds:RS_164._col1=RS_165._col0(Inner),Output:["_col6"] - <-Map 46 [SIMPLE_EDGE] + <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_165] PartitionCols:_col0 Select Operator [SEL_93] (rows=80000000 width=860) @@ -201,12 +188,12 @@ Stage-0 predicate:c_customer_sk is not null TableScan [TS_91] (rows=80000000 width=860) default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"] - <-Reducer 26 [SIMPLE_EDGE] + <-Reducer 18 [SIMPLE_EDGE] SHUFFLE [RS_164] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_360] (rows=633595212 width=88) Conds:RS_161._col0=RS_162._col0(Inner),Output:["_col1"] - <-Map 31 [SIMPLE_EDGE] + <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_162] PartitionCols:_col0 Select Operator [SEL_38] (rows=36525 width=1119) @@ -215,7 +202,7 @@ Stage-0 predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null) TableScan [TS_36] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Map 20 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_161] PartitionCols:_col0 Select Operator [SEL_35] (rows=575995635 width=88) @@ -224,20 +211,20 @@ Stage-0 predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) TableScan [TS_33] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"] - <-Reducer 39 [CUSTOM_SIMPLE_EDGE] + <-Reducer 28 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_223] Group By Operator [GBY_205] (rows=1 width=224) Output:["_col0"],aggregations:["max(VALUE._col0)"] - <-Reducer 38 [CUSTOM_SIMPLE_EDGE] + <-Reducer 27 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_204] - Group By Operator [GBY_203] (rows=1 width=224) + Group By Operator [GBY_84] (rows=1 width=224) Output:["_col0"],aggregations:["max(_col1)"] - Select Operator [SEL_201] (rows=348477374 width=88) + Select Operator [SEL_82] (rows=348477374 width=88) Output:["_col1"] - Group By Operator [GBY_200] (rows=348477374 width=88) + Group By Operator [GBY_81] (rows=348477374 width=88) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_199] + <-Reducer 26 [SIMPLE_EDGE] + SHUFFLE [RS_80] PartitionCols:_col0 Group By Operator [GBY_198] (rows=696954748 width=88) Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 @@ -245,20 +232,20 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_363] (rows=696954748 width=88) Conds:RS_193._col1=RS_194._col0(Inner),Output:["_col2","_col3","_col6"] - <-Map 46 [SIMPLE_EDGE] + <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_194] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_93] - <-Reducer 36 [SIMPLE_EDGE] + <-Reducer 25 [SIMPLE_EDGE] SHUFFLE [RS_193] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_362] (rows=633595212 width=88) Conds:RS_190._col0=RS_191._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 31 [SIMPLE_EDGE] + <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_191] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_38] - <-Map 40 [SIMPLE_EDGE] + <-Map 29 [SIMPLE_EDGE] SHUFFLE [RS_190] PartitionCols:_col0 Select Operator [SEL_64] (rows=575995635 width=88) @@ -267,12 +254,12 @@ Stage-0 predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null) TableScan [TS_62] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_quantity","ss_sales_price"] - <-Reducer 45 [CUSTOM_SIMPLE_EDGE] + <-Reducer 32 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_224] - Group By Operator [GBY_220] (rows=316797606 width=88) + Group By Operator [GBY_101] (rows=316797606 width=88) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 44 [SIMPLE_EDGE] - SHUFFLE [RS_219] + <-Reducer 31 [SIMPLE_EDGE] + SHUFFLE [RS_100] PartitionCols:_col0 Group By Operator [GBY_218] (rows=633595212 width=88) Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 @@ -280,11 +267,11 @@ Stage-0 Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_364] (rows=633595212 width=88) Conds:RS_213._col0=RS_214._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 46 [SIMPLE_EDGE] + <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_214] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_93] - <-Map 41 [SIMPLE_EDGE] + <-Map 30 [SIMPLE_EDGE] SHUFFLE [RS_213] PartitionCols:_col0 Select Operator [SEL_90] (rows=575995635 width=88) @@ -298,7 +285,7 @@ Stage-0 PartitionCols:_col2 Merge Join Operator [MERGEJOIN_366] (rows=174243235 width=135) Conds:RS_231._col1=RS_232._col0(Inner),Output:["_col2","_col3","_col4"] - <-Reducer 17 [SIMPLE_EDGE] + <-Reducer 14 [SIMPLE_EDGE] SHUFFLE [RS_232] PartitionCols:_col0 Group By Operator [GBY_150] (rows=58079562 width=88) @@ -311,17 +298,17 @@ Stage-0 Output:["_col0","_col3"] Group By Operator [GBY_144] (rows=348477374 width=88) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 16 [SIMPLE_EDGE] + <-Reducer 13 [SIMPLE_EDGE] SHUFFLE [RS_143] PartitionCols:_col0 - Group By Operator [GBY_142] (rows=696954748 width=88) + Group By Operator [GBY_23] (rows=696954748 width=88) Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col1, _col0, _col2 - Select Operator [SEL_140] (rows=696954748 width=88) + Select Operator [SEL_21] (rows=696954748 width=88) Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_359] (rows=696954748 width=88) - Conds:RS_137._col1=RS_138._col0(Inner),Output:["_col3","_col5","_col6"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_138] + Merge Join Operator [MERGEJOIN_351] (rows=696954748 width=88) + Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col5","_col6"] + <-Map 16 [SIMPLE_EDGE] + SHUFFLE [RS_19] PartitionCols:_col0 Select Operator [SEL_14] (rows=462000 width=1436) Output:["_col0","_col1"] @@ -329,13 +316,13 @@ Stage-0 predicate:i_item_sk is not null TableScan [TS_12] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_137] + <-Reducer 12 [SIMPLE_EDGE] + SHUFFLE [RS_18] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_358] (rows=633595212 width=88) - Conds:RS_134._col0=RS_135._col0(Inner),Output:["_col1","_col3"] + Merge Join Operator [MERGEJOIN_350] (rows=633595212 width=88) + Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col3"] <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_134] + SHUFFLE [RS_15] PartitionCols:_col0 Select Operator [SEL_8] (rows=575995635 width=88) Output:["_col0","_col1"] @@ -343,8 +330,8 @@ Stage-0 predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) TableScan [TS_6] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_135] + <-Map 15 [SIMPLE_EDGE] + SHUFFLE [RS_16] PartitionCols:_col0 Select Operator [SEL_11] (rows=36525 width=1119) Output:["_col0","_col1"] @@ -366,7 +353,7 @@ Stage-0 predicate:((d_moy = 1) and (d_year = 1999) and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 47 [SIMPLE_EDGE] + <-Map 34 [SIMPLE_EDGE] SHUFFLE [RS_228] PartitionCols:_col0 Select Operator [SEL_121] (rows=144002668 width=135) @@ -383,7 +370,7 @@ Stage-0 Output:["_col0"] Merge Join Operator [MERGEJOIN_368] (rows=383314495 width=135) Conds:RS_115._col1=RS_116._col0(Inner),Output:["_col3","_col4"] - <-Reducer 25 [SIMPLE_EDGE] + <-Reducer 22 [SIMPLE_EDGE] SHUFFLE [RS_116] PartitionCols:_col0 Select Operator [SEL_108] (rows=105599202 width=321) @@ -392,105 +379,15 @@ Stage-0 predicate:(_col3 > (0.95 * _col1)) Merge Join Operator [MERGEJOIN_367] (rows=316797606 width=321) Conds:(Inner),(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 24 [CUSTOM_SIMPLE_EDGE] + <-Reducer 21 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_103] - Select Operator [SEL_61] (rows=1 width=8) - Filter Operator [FIL_60] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_58] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 23 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_57] - Group By Operator [GBY_56] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_53] (rows=348477374 width=88) - Group By Operator [GBY_52] (rows=348477374 width=88) - Output:["_col0"],keys:KEY._col0 - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col0 - Group By Operator [GBY_50] (rows=696954748 width=88) - Output:["_col0"],keys:_col0 - Select Operator [SEL_48] (rows=696954748 width=88) - Output:["_col0"] - Merge Join Operator [MERGEJOIN_353] (rows=696954748 width=88) - Conds:RS_45._col1=RS_46._col0(Inner),Output:["_col6"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_93] - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_352] (rows=633595212 width=88) - Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col1"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_38] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_42] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_35] - <-Reducer 35 [CUSTOM_SIMPLE_EDGE] + Please refer to the previous Select Operator [SEL_61] + <-Reducer 28 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_104] - Group By Operator [GBY_86] (rows=1 width=224) - Output:["_col0"],aggregations:["max(VALUE._col0)"] - <-Reducer 34 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_85] - Group By Operator [GBY_84] (rows=1 width=224) - Output:["_col0"],aggregations:["max(_col1)"] - Select Operator [SEL_82] (rows=348477374 width=88) - Output:["_col1"] - Group By Operator [GBY_81] (rows=348477374 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 33 [SIMPLE_EDGE] - SHUFFLE [RS_80] - PartitionCols:_col0 - Group By Operator [GBY_79] (rows=696954748 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_77] (rows=696954748 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_355] (rows=696954748 width=88) - Conds:RS_74._col1=RS_75._col0(Inner),Output:["_col2","_col3","_col6"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_93] - <-Reducer 32 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_354] (rows=633595212 width=88) - Conds:RS_71._col0=RS_72._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_72] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_38] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_71] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_64] - <-Reducer 43 [CUSTOM_SIMPLE_EDGE] + Please refer to the previous Group By Operator [GBY_205] + <-Reducer 32 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_105] - Group By Operator [GBY_101] (rows=316797606 width=88) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 42 [SIMPLE_EDGE] - SHUFFLE [RS_100] - PartitionCols:_col0 - Group By Operator [GBY_99] (rows=633595212 width=88) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Select Operator [SEL_97] (rows=633595212 width=88) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_356] (rows=633595212 width=88) - Conds:RS_94._col0=RS_95._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 46 [SIMPLE_EDGE] - SHUFFLE [RS_95] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_93] - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_94] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_90] + Please refer to the previous Group By Operator [GBY_101] <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_115] PartitionCols:_col1 @@ -499,42 +396,7 @@ Stage-0 <-Reducer 14 [SIMPLE_EDGE] SHUFFLE [RS_113] PartitionCols:_col0 - Group By Operator [GBY_31] (rows=58079562 width=88) - Output:["_col0"],keys:_col1 - Select Operator [SEL_27] (rows=116159124 width=88) - Output:["_col1"] - Filter Operator [FIL_26] (rows=116159124 width=88) - predicate:(_col3 > 4) - Select Operator [SEL_347] (rows=348477374 width=88) - Output:["_col0","_col3"] - Group By Operator [GBY_25] (rows=348477374 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_24] - PartitionCols:_col0 - Group By Operator [GBY_23] (rows=696954748 width=88) - Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col1, _col0, _col2 - Select Operator [SEL_21] (rows=696954748 width=88) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_351] (rows=696954748 width=88) - Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col5","_col6"] - <-Map 19 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_14] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_350] (rows=633595212 width=88) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col3"] - <-Map 11 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_8] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_11] + Please refer to the previous Group By Operator [GBY_150] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_112] PartitionCols:_col2 diff --git a/ql/src/test/results/clientpositive/perf/tez/query32.q.out b/ql/src/test/results/clientpositive/perf/tez/query32.q.out index bba1bd8468..3ad9595910 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query32.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query32.q.out @@ -55,12 +55,11 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 9 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) +Reducer 5 <- Reducer 2 (SIMPLE_EDGE) +Reducer 6 <- Map 8 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Stage-0 Fetch Operator @@ -96,7 +95,7 @@ Stage-0 predicate:(cs_item_sk is not null and cs_sold_date_sk is not null) TableScan [TS_0] (rows=287989836 width=135) default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_discount_amt"] - <-Map 8 [SIMPLE_EDGE] + <-Map 7 [SIMPLE_EDGE] SHUFFLE [RS_28] PartitionCols:_col0 Select Operator [SEL_5] (rows=8116 width=1119) @@ -105,12 +104,12 @@ Stage-0 predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-03-18 00:00:00.0 AND 1998-06-16 01:00:00.0 and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 7 [ONE_TO_ONE_EDGE] + <-Reducer 6 [ONE_TO_ONE_EDGE] FORWARD [RS_31] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_60] (rows=174233858 width=135) Conds:RS_23._col1=RS_24._col0(Inner),Output:["_col0","_col2"] - <-Map 9 [SIMPLE_EDGE] + <-Map 8 [SIMPLE_EDGE] SHUFFLE [RS_24] PartitionCols:_col0 Select Operator [SEL_22] (rows=231000 width=1436) @@ -119,26 +118,17 @@ Stage-0 predicate:((i_manufact_id = 269) and i_item_sk is not null) TableScan [TS_20] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 6 [ONE_TO_ONE_EDGE] + <-Reducer 5 [ONE_TO_ONE_EDGE] FORWARD [RS_23] PartitionCols:_col1 Select Operator [SEL_19] (rows=158394413 width=135) Output:["_col0","_col1"] Group By Operator [GBY_18] (rows=158394413 width=135) Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 5 [SIMPLE_EDGE] + <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_17] PartitionCols:_col0 Group By Operator [GBY_16] (rows=316788826 width=135) Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col1 - Merge Join Operator [MERGEJOIN_59] (rows=316788826 width=135) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_2] - <-Map 8 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_5] + Please refer to the previous Merge Join Operator [MERGEJOIN_58] diff --git a/ql/src/test/results/clientpositive/perf/tez/query33.q.out b/ql/src/test/results/clientpositive/perf/tez/query33.q.out index 0810c28a57..9b5e5de8c9 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query33.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query33.q.out @@ -149,26 +149,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (ONE_TO_ONE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 15 <- Map 14 (SIMPLE_EDGE) -Reducer 16 <- Map 14 (SIMPLE_EDGE) -Reducer 17 <- Map 14 (SIMPLE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE) -Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE) -Reducer 23 <- Map 26 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 24 <- Map 21 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE) -Reducer 25 <- Map 26 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) +Reducer 10 <- Reducer 2 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) +Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Union 5 (CONTAINS) +Reducer 13 <- Map 12 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) +Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) +Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) +Reducer 19 <- Map 22 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE) +Reducer 20 <- Map 17 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) +Reducer 21 <- Map 22 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) +Reducer 3 <- Reducer 19 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (ONE_TO_ONE_EDGE) -Reducer 9 <- Reducer 23 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 8 <- Reducer 16 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 9 <- Reducer 8 (SIMPLE_EDGE), Union 5 (CONTAINS) Stage-0 Fetch Operator @@ -185,61 +181,22 @@ Stage-0 Group By Operator [GBY_116] (rows=335408073 width=108) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 <-Union 5 [SIMPLE_EDGE] - <-Reducer 10 [CONTAINS] + <-Reducer 11 [CONTAINS] Reduce Output Operator [RS_115] PartitionCols:_col0 Group By Operator [GBY_114] (rows=670816147 width=108) Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_72] (rows=191657247 width=135) + Group By Operator [GBY_110] (rows=95833781 width=135) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_71] + <-Reducer 10 [SIMPLE_EDGE] + SHUFFLE [RS_109] PartitionCols:_col0 - Group By Operator [GBY_70] (rows=383314495 width=135) + Group By Operator [GBY_108] (rows=191667562 width=135) Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135) - Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col4 - Select Operator [SEL_62] (rows=348467716 width=135) - Output:["_col4","_col5"] - Merge Join Operator [MERGEJOIN_178] (rows=348467716 width=135) - Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_162] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_177] (rows=316788826 width=135) - Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_161] (rows=18262 width=1119) - predicate:((d_moy = 3) and (d_year = 1999) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_165] (rows=287989836 width=135) - predicate:(cs_bill_addr_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_47] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_66] + Merge Join Operator [MERGEJOIN_184] (rows=191667562 width=135) + Conds:RS_104._col0=RS_105._col3(Inner),Output:["_col1","_col8"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_104] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_176] (rows=508200 width=1436) Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] @@ -252,12 +209,12 @@ Stage-0 predicate:(i_item_sk is not null and i_manufact_id is not null) TableScan [TS_0] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 16 [ONE_TO_ONE_EDGE] + <-Reducer 13 [ONE_TO_ONE_EDGE] FORWARD [RS_64] PartitionCols:_col0 Group By Operator [GBY_45] (rows=115500 width=1436) Output:["_col0"],keys:KEY._col0 - <-Map 14 [SIMPLE_EDGE] + <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_44] PartitionCols:_col0 Group By Operator [GBY_6] (rows=231000 width=1436) @@ -268,59 +225,37 @@ Stage-0 predicate:((i_category) IN ('Books') and i_manufact_id is not null) TableScan [TS_3] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"] - <-Reducer 13 [CONTAINS] - Reduce Output Operator [RS_115] - PartitionCols:_col0 - Group By Operator [GBY_114] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_110] (rows=95833781 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Group By Operator [GBY_108] (rows=191667562 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=191667562 width=135) - Conds:RS_104._col0=RS_105._col3(Inner),Output:["_col1","_col8"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_179] (rows=508200 width=1436) - Conds:RS_101._col1=RS_102._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_2] - <-Reducer 17 [ONE_TO_ONE_EDGE] - FORWARD [RS_102] - PartitionCols:_col0 - Group By Operator [GBY_83] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_6] - <-Reducer 25 [SIMPLE_EDGE] + <-Reducer 21 [SIMPLE_EDGE] SHUFFLE [RS_105] PartitionCols:_col3 Select Operator [SEL_100] (rows=174243235 width=135) Output:["_col3","_col5"] Merge Join Operator [MERGEJOIN_181] (rows=174243235 width=135) Conds:RS_97._col2=RS_98._col0(Inner),Output:["_col1","_col3"] - <-Map 26 [SIMPLE_EDGE] + <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_98] PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_18] - <-Reducer 24 [SIMPLE_EDGE] + Select Operator [SEL_18] (rows=20000000 width=1014) + Output:["_col0"] + Filter Operator [FIL_162] (rows=20000000 width=1014) + predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) + TableScan [TS_16] (rows=40000000 width=1014) + default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] + <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_97] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_180] (rows=158402938 width=135) Conds:RS_94._col0=RS_95._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_95] PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_15] - <-Map 28 [SIMPLE_EDGE] + Select Operator [SEL_15] (rows=18262 width=1119) + Output:["_col0"] + Filter Operator [FIL_161] (rows=18262 width=1119) + predicate:((d_moy = 3) and (d_year = 1999) and d_date_sk is not null) + TableScan [TS_13] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0 Select Operator [SEL_87] (rows=144002668 width=135) @@ -334,9 +269,56 @@ Stage-0 PartitionCols:_col0 Group By Operator [GBY_114] (rows=670816147 width=108) Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_35] (rows=383325119 width=88) + Group By Operator [GBY_72] (rows=191657247 width=135) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_71] + PartitionCols:_col0 + Group By Operator [GBY_70] (rows=383314495 width=135) + Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 + Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135) + Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_66] + PartitionCols:_col0 + Please refer to the previous Merge Join Operator [MERGEJOIN_176] + <-Reducer 19 [SIMPLE_EDGE] + SHUFFLE [RS_67] + PartitionCols:_col4 + Select Operator [SEL_62] (rows=348467716 width=135) + Output:["_col4","_col5"] + Merge Join Operator [MERGEJOIN_178] (rows=348467716 width=135) + Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] + <-Map 22 [SIMPLE_EDGE] + SHUFFLE [RS_60] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_18] + <-Reducer 18 [SIMPLE_EDGE] + SHUFFLE [RS_59] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_177] (rows=316788826 width=135) + Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] + <-Map 17 [SIMPLE_EDGE] + SHUFFLE [RS_57] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_15] + <-Map 23 [SIMPLE_EDGE] + SHUFFLE [RS_56] + PartitionCols:_col0 + Select Operator [SEL_49] (rows=287989836 width=135) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_165] (rows=287989836 width=135) + predicate:(cs_bill_addr_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) + TableScan [TS_47] (rows=287989836 width=135) + default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] + <-Reducer 9 [CONTAINS] + Reduce Output Operator [RS_115] + PartitionCols:_col0 + Group By Operator [GBY_114] (rows=670816147 width=108) + Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 + Group By Operator [GBY_35] (rows=383325119 width=88) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 + <-Reducer 8 [SIMPLE_EDGE] SHUFFLE [RS_34] PartitionCols:_col0 Group By Operator [GBY_33] (rows=766650239 width=88) @@ -346,42 +328,28 @@ Stage-0 <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_29] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_173] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_2] - <-Reducer 15 [ONE_TO_ONE_EDGE] - FORWARD [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_6] - <-Reducer 20 [SIMPLE_EDGE] + Please refer to the previous Merge Join Operator [MERGEJOIN_176] + <-Reducer 16 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col3 Select Operator [SEL_25] (rows=696954748 width=88) Output:["_col3","_col5"] Merge Join Operator [MERGEJOIN_175] (rows=696954748 width=88) Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 26 [SIMPLE_EDGE] + <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_23] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_18] - <-Reducer 19 [SIMPLE_EDGE] + <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_174] (rows=633595212 width=88) Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_20] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_15] - <-Map 18 [SIMPLE_EDGE] + <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 Select Operator [SEL_12] (rows=575995635 width=88) diff --git a/ql/src/test/results/clientpositive/perf/tez/query44.q.out b/ql/src/test/results/clientpositive/perf/tez/query44.q.out index 07187b14d4..50dc1bc122 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query44.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query44.q.out @@ -1,5 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[131][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 9' is a cross product -Warning: Shuffle Join MERGEJOIN[133][tables = [$hdt$_4, $hdt$_5, $hdt$_3]] in Stage 'Reducer 13' is a cross product +Warning: Shuffle Join MERGEJOIN[133][tables = [$hdt$_4, $hdt$_5, $hdt$_3]] in Stage 'Reducer 9' is a cross product PREHOOK: query: explain select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * @@ -72,21 +71,16 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 6 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (CUSTOM_SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (CUSTOM_SIMPLE_EDGE), Reducer 17 (CUSTOM_SIMPLE_EDGE), Reducer 20 (CUSTOM_SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE) -Reducer 16 <- Map 15 (SIMPLE_EDGE) -Reducer 17 <- Map 15 (SIMPLE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 20 <- Map 18 (SIMPLE_EDGE) +Reducer 11 <- Reducer 9 (SIMPLE_EDGE) +Reducer 13 <- Map 12 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 1 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) +Reducer 5 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) -Reducer 9 <- Reducer 16 (CUSTOM_SIMPLE_EDGE), Reducer 19 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) +Reducer 9 <- Reducer 13 (CUSTOM_SIMPLE_EDGE), Reducer 15 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) Stage-0 Fetch Operator @@ -118,7 +112,7 @@ Stage-0 predicate:i_item_sk is not null TableScan [TS_0] (rows=462000 width=1436) default@item,i1,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_product_name"] - <-Reducer 10 [SIMPLE_EDGE] + <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_95] PartitionCols:_col0 Select Operator [SEL_41] (rows=1151982528066248 width=185) @@ -132,17 +126,17 @@ Stage-0 <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:0 - Filter Operator [FIL_37] (rows=3455947584198744 width=185) + Filter Operator [FIL_82] (rows=3455947584198744 width=185) predicate:(_col3 > (0.9 * _col1)) - Merge Join Operator [MERGEJOIN_131] (rows=10367842752596232 width=185) + Merge Join Operator [MERGEJOIN_133] (rows=10367842752596232 width=185) Conds:(Inner),(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 16 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_34] + <-Reducer 13 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_79] Select Operator [SEL_25] (rows=71999454 width=88) Output:["_col0"] Group By Operator [GBY_24] (rows=71999454 width=88) Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Map 15 [SIMPLE_EDGE] + <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_23] PartitionCols:_col0 Group By Operator [GBY_22] (rows=143998908 width=88) @@ -153,11 +147,11 @@ Stage-0 predicate:((ss_store_sk = 410) and ss_hdemo_sk is null) TableScan [TS_18] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_hdemo_sk","ss_store_sk","ss_net_profit"] - <-Reducer 19 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_35] + <-Reducer 15 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_80] Group By Operator [GBY_31] (rows=143998908 width=88) Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Map 18 [SIMPLE_EDGE] + <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col0 Group By Operator [GBY_29] (rows=287997817 width=88) @@ -169,21 +163,21 @@ Stage-0 TableScan [TS_26] (rows=575995635 width=88) default@store_sales,ss1,Tbl:COMPLETE,Col:NONE,Output:["ss_item_sk","ss_store_sk","ss_net_profit"] <-Reducer 8 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_33] - Select Operator [SEL_17] (rows=1 width=8) - Filter Operator [FIL_16] (rows=1 width=8) + PARTITION_ONLY_SHUFFLE [RS_78] + Select Operator [SEL_62] (rows=1 width=8) + Filter Operator [FIL_61] (rows=1 width=8) predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_14] (rows=1 width=8) + Group By Operator [GBY_59] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 7 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_13] - Group By Operator [GBY_12] (rows=1 width=8) + PARTITION_ONLY_SHUFFLE [RS_58] + Group By Operator [GBY_57] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_10] (rows=71999454 width=88) - Group By Operator [GBY_9] (rows=71999454 width=88) + Select Operator [SEL_55] (rows=71999454 width=88) + Group By Operator [GBY_54] (rows=71999454 width=88) Output:["_col0"],keys:KEY._col0 <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_8] + SHUFFLE [RS_53] PartitionCols:_col0 Group By Operator [GBY_7] (rows=143998908 width=88) Output:["_col0"],keys:410 @@ -201,7 +195,7 @@ Stage-0 SHUFFLE [RS_90] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_2] - <-Reducer 14 [SIMPLE_EDGE] + <-Reducer 10 [SIMPLE_EDGE] SHUFFLE [RS_91] PartitionCols:_col0 Select Operator [SEL_86] (rows=1151982528066248 width=185) @@ -212,47 +206,8 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 DESC NULLS LAST","partition by:":"0"}] Select Operator [SEL_84] (rows=3455947584198744 width=185) Output:["_col2","_col3"] - <-Reducer 13 [SIMPLE_EDGE] + <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_83] PartitionCols:0 - Filter Operator [FIL_82] (rows=3455947584198744 width=185) - predicate:(_col3 > (0.9 * _col1)) - Merge Join Operator [MERGEJOIN_133] (rows=10367842752596232 width=185) - Conds:(Inner),(Inner),Output:["_col1","_col2","_col3"] - <-Reducer 12 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_78] - Select Operator [SEL_62] (rows=1 width=8) - Filter Operator [FIL_61] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_59] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 11 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_58] - Group By Operator [GBY_57] (rows=1 width=8) - Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_55] (rows=71999454 width=88) - Group By Operator [GBY_54] (rows=71999454 width=88) - Output:["_col0"],keys:KEY._col0 - <-Map 6 [SIMPLE_EDGE] - SHUFFLE [RS_53] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_7] - <-Reducer 17 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_79] - Select Operator [SEL_70] (rows=71999454 width=88) - Output:["_col0"] - Group By Operator [GBY_69] (rows=71999454 width=88) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Map 15 [SIMPLE_EDGE] - SHUFFLE [RS_68] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_22] - <-Reducer 20 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_80] - Group By Operator [GBY_76] (rows=143998908 width=88) - Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_75] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_29] + Please refer to the previous Filter Operator [FIL_82] diff --git a/ql/src/test/results/clientpositive/perf/tez/query47.q.out b/ql/src/test/results/clientpositive/perf/tez/query47.q.out index 128840872e..ff99e64d00 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query47.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query47.q.out @@ -101,68 +101,60 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 21 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 22 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 15 <- Map 21 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 3 <- Map 21 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 22 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 10 <- Reducer 5 (SIMPLE_EDGE) +Reducer 11 <- Reducer 5 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) +Reducer 3 <- Map 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 4 <- Map 14 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 13 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) +Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +Reducer 8 <- Reducer 10 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Reducer 9 <- Reducer 8 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 8 + Reducer 9 File Output Operator [FS_112] Limit [LIM_110] (rows=100 width=88) Number of rows:100 Select Operator [SEL_109] (rows=843315280 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 7 [SIMPLE_EDGE] + <-Reducer 8 [SIMPLE_EDGE] SHUFFLE [RS_108] Select Operator [SEL_107] (rows=843315280 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] Merge Join Operator [MERGEJOIN_189] (rows=843315280 width=88) Conds:RS_103._col0, _col1, _col2, _col3, (_col7 + 1)=RS_104._col0, _col1, _col2, _col3, _col8(Inner),RS_104._col0, _col1, _col2, _col3, _col8=RS_105._col0, _col1, _col2, _col3, (_col7 - 1)(Inner),Output:["_col6","_col8","_col12","_col13","_col14","_col15","_col23"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0, _col1, _col2, _col3, (_col7 + 1) - Select Operator [SEL_29] (rows=383325119 width=88) + <-Reducer 10 [SIMPLE_EDGE] + SHUFFLE [RS_105] + PartitionCols:_col0, _col1, _col2, _col3, (_col7 - 1) + Select Operator [SEL_99] (rows=383325119 width=88) Output:["_col0","_col1","_col2","_col3","_col6","_col7"] - Filter Operator [FIL_164] (rows=383325119 width=88) + Filter Operator [FIL_175] (rows=383325119 width=88) predicate:rank_window_0 is not null - PTF Operator [PTF_28] (rows=383325119 width=88) + PTF Operator [PTF_98] (rows=383325119 width=88) Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3"}] - Select Operator [SEL_27] (rows=383325119 width=88) + Select Operator [SEL_97] (rows=383325119 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_26] + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_96] PartitionCols:_col0, _col1, _col2, _col3 - Select Operator [SEL_25] (rows=383325119 width=88) + Select Operator [SEL_58] (rows=383325119 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_24] (rows=383325119 width=88) + Group By Operator [GBY_57] (rows=383325119 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_23] + <-Reducer 4 [SIMPLE_EDGE] + SHUFFLE [RS_56] PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_22] (rows=766650239 width=88) + Group By Operator [GBY_55] (rows=766650239 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col9, _col11, _col12 - Merge Join Operator [MERGEJOIN_182] (rows=766650239 width=88) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col11","_col12"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_19] + Merge Join Operator [MERGEJOIN_185] (rows=766650239 width=88) + Conds:RS_51._col2=RS_52._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col11","_col12"] + <-Map 14 [SIMPLE_EDGE] + SHUFFLE [RS_52] PartitionCols:_col0 Select Operator [SEL_81] (rows=1704 width=1910) Output:["_col0","_col1","_col2"] @@ -170,12 +162,12 @@ Stage-0 predicate:(s_company_name is not null and s_store_name is not null and s_store_sk is not null) TableScan [TS_79] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_name"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_18] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_51] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_181] (rows=696954748 width=88) Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9"] - <-Map 21 [SIMPLE_EDGE] + <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_16] PartitionCols:_col0 Select Operator [SEL_78] (rows=462000 width=1436) @@ -184,7 +176,7 @@ Stage-0 predicate:(i_brand is not null and i_category is not null and i_item_sk is not null) TableScan [TS_76] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"] - <-Reducer 9 [SIMPLE_EDGE] + <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_15] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_180] (rows=633595212 width=88) @@ -198,7 +190,7 @@ Stage-0 predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null) TableScan [TS_70] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"] - <-Map 20 [SIMPLE_EDGE] + <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_13] PartitionCols:_col0 Select Operator [SEL_75] (rows=73048 width=1119) @@ -207,7 +199,22 @@ Stage-0 predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null) TableScan [TS_73] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 19 [SIMPLE_EDGE] + <-Reducer 11 [SIMPLE_EDGE] + SHUFFLE [RS_103] + PartitionCols:_col0, _col1, _col2, _col3, (_col7 + 1) + Select Operator [SEL_29] (rows=383325119 width=88) + Output:["_col0","_col1","_col2","_col3","_col6","_col7"] + Filter Operator [FIL_164] (rows=383325119 width=88) + predicate:rank_window_0 is not null + PTF Operator [PTF_28] (rows=383325119 width=88) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3"}] + Select Operator [SEL_27] (rows=383325119 width=88) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_26] + PartitionCols:_col0, _col1, _col2, _col3 + Please refer to the previous Select Operator [SEL_58] + <-Reducer 7 [SIMPLE_EDGE] SHUFFLE [RS_104] PartitionCols:_col0, _col1, _col2, _col3, _col8 Select Operator [SEL_67] (rows=31943759 width=88) @@ -222,7 +229,7 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS FIRST, _col6 ASC NULLS FIRST","partition by:":"_col1, _col2, _col3, _col4"}] Select Operator [SEL_64] (rows=383325119 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Reducer 18 [SIMPLE_EDGE] + <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_63] PartitionCols:_col0, _col1, _col2, _col3 Select Operator [SEL_62] (rows=383325119 width=88) @@ -231,95 +238,8 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3, _col4"}] Select Operator [SEL_60] (rows=383325119 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 17 [SIMPLE_EDGE] + <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_59] PartitionCols:_col0, _col1, _col2, _col3, _col4 - Select Operator [SEL_58] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_57] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_55] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col9, _col11, _col12 - Merge Join Operator [MERGEJOIN_185] (rows=766650239 width=88) - Conds:RS_51._col2=RS_52._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col11","_col12"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_52] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_81] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_51] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_184] (rows=696954748 width=88) - Conds:RS_48._col1=RS_49._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_49] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_78] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_48] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=633595212 width=88) - Conds:RS_45._col0=RS_46._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_45] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_72] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_46] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_75] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col0, _col1, _col2, _col3, (_col7 - 1) - Select Operator [SEL_99] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col6","_col7"] - Filter Operator [FIL_175] (rows=383325119 width=88) - predicate:rank_window_0 is not null - PTF Operator [PTF_98] (rows=383325119 width=88) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3"}] - Select Operator [SEL_97] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_96] - PartitionCols:_col0, _col1, _col2, _col3 - Select Operator [SEL_95] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - Group By Operator [GBY_94] (rows=383325119 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_92] (rows=766650239 width=88) - Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col9, _col11, _col12 - Merge Join Operator [MERGEJOIN_188] (rows=766650239 width=88) - Conds:RS_88._col2=RS_89._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col11","_col12"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_81] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_88] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_187] (rows=696954748 width=88) - Conds:RS_85._col1=RS_86._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_86] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_78] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_85] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_186] (rows=633595212 width=88) - Conds:RS_82._col0=RS_83._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_72] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_75] + Please refer to the previous Select Operator [SEL_58] diff --git a/ql/src/test/results/clientpositive/perf/tez/query54.q.out b/ql/src/test/results/clientpositive/perf/tez/query54.q.out index e6b9b28bd0..8bbfebdbec 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query54.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query54.q.out @@ -1,6 +1,6 @@ Warning: Shuffle Join MERGEJOIN[184][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product Warning: Shuffle Join MERGEJOIN[185][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[183][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 30' is a cross product +Warning: Shuffle Join MERGEJOIN[183][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 29' is a cross product Warning: Shuffle Join MERGEJOIN[186][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 6' is a cross product PREHOOK: query: explain with my_customers as ( @@ -127,14 +127,13 @@ Reducer 20 <- Reducer 19 (SIMPLE_EDGE) Reducer 26 <- Map 25 (SIMPLE_EDGE) Reducer 27 <- Reducer 26 (CUSTOM_SIMPLE_EDGE) Reducer 28 <- Map 25 (SIMPLE_EDGE) -Reducer 29 <- Map 25 (SIMPLE_EDGE) +Reducer 29 <- Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 31 (CUSTOM_SIMPLE_EDGE) Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (CUSTOM_SIMPLE_EDGE), Reducer 32 (CUSTOM_SIMPLE_EDGE) -Reducer 31 <- Map 25 (SIMPLE_EDGE) -Reducer 32 <- Reducer 31 (CUSTOM_SIMPLE_EDGE) +Reducer 30 <- Map 25 (SIMPLE_EDGE) +Reducer 31 <- Reducer 30 (CUSTOM_SIMPLE_EDGE) Reducer 4 <- Reducer 27 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Reducer 30 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) +Reducer 5 <- Reducer 26 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) +Reducer 6 <- Reducer 29 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE) @@ -177,11 +176,11 @@ Stage-0 Output:["_col0","_col4","_col11","_col13","_col15"] Merge Join Operator [MERGEJOIN_186] (rows=58108714324214428 width=158) Conds:(Inner),Output:["_col0","_col2","_col6","_col13","_col15"] - <-Reducer 30 [CUSTOM_SIMPLE_EDGE] + <-Reducer 29 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_112] Merge Join Operator [MERGEJOIN_183] (rows=9131 width=1128) Conds:(Right Outer),Output:["_col0"] - <-Reducer 29 [CUSTOM_SIMPLE_EDGE] + <-Reducer 28 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_109] Group By Operator [GBY_6] (rows=9131 width=1119) Output:["_col0"],keys:KEY._col0 @@ -196,14 +195,14 @@ Stage-0 predicate:((d_moy = 3) and (d_year = 1999)) TableScan [TS_73] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"] - <-Reducer 32 [CUSTOM_SIMPLE_EDGE] + <-Reducer 31 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_110] Select Operator [SEL_22] (rows=1 width=8) Filter Operator [FIL_21] (rows=1 width=8) predicate:(sq_count_check(_col0) <= 1) Group By Operator [GBY_19] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 31 [CUSTOM_SIMPLE_EDGE] + <-Reducer 30 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_18] Group By Operator [GBY_17] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] @@ -224,7 +223,7 @@ Stage-0 Output:["_col0","_col4","_col11","_col13"] Merge Join Operator [MERGEJOIN_185] (rows=6363893803988 width=1217) Conds:(Left Outer),Output:["_col2","_col4","_col10","_col13"] - <-Reducer 28 [CUSTOM_SIMPLE_EDGE] + <-Reducer 26 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_106] Group By Operator [GBY_94] (rows=9131 width=1119) Output:["_col0"],keys:KEY._col0 @@ -252,12 +251,7 @@ Stage-0 Group By Operator [GBY_82] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Select Operator [SEL_80] (rows=9131 width=1119) - Group By Operator [GBY_79] (rows=9131 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_78] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_77] + Please refer to the previous Group By Operator [GBY_94] <-Reducer 3 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_102] Merge Join Operator [MERGEJOIN_182] (rows=696954748 width=88) diff --git a/ql/src/test/results/clientpositive/perf/tez/query56.q.out b/ql/src/test/results/clientpositive/perf/tez/query56.q.out index 40417405a3..642046fbe8 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query56.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query56.q.out @@ -135,26 +135,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (ONE_TO_ONE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 15 <- Map 14 (SIMPLE_EDGE) -Reducer 16 <- Map 14 (SIMPLE_EDGE) -Reducer 17 <- Map 14 (SIMPLE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE) -Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE) -Reducer 23 <- Map 26 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 24 <- Map 21 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE) -Reducer 25 <- Map 26 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) +Reducer 10 <- Reducer 19 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Union 5 (CONTAINS) +Reducer 13 <- Map 12 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) +Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) +Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) +Reducer 19 <- Map 22 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE) +Reducer 20 <- Map 17 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) +Reducer 21 <- Map 22 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (ONE_TO_ONE_EDGE) -Reducer 9 <- Reducer 23 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 8 <- Reducer 16 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 9 <- Reducer 8 (SIMPLE_EDGE), Union 5 (CONTAINS) Stage-0 Fetch Operator @@ -171,28 +167,58 @@ Stage-0 Group By Operator [GBY_116] (rows=335408073 width=108) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 <-Union 5 [SIMPLE_EDGE] - <-Reducer 10 [CONTAINS] + <-Reducer 11 [CONTAINS] Reduce Output Operator [RS_115] PartitionCols:_col0 Group By Operator [GBY_114] (rows=670816147 width=108) Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 Group By Operator [GBY_72] (rows=191657247 width=135) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 9 [SIMPLE_EDGE] + <-Reducer 10 [SIMPLE_EDGE] SHUFFLE [RS_71] PartitionCols:_col0 Group By Operator [GBY_70] (rows=383314495 width=135) Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135) Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 23 [SIMPLE_EDGE] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_66] + PartitionCols:_col0 + Merge Join Operator [MERGEJOIN_179] (rows=508200 width=1436) + Conds:RS_101._col1=RS_102._col0(Inner),Output:["_col0","_col1"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_101] + PartitionCols:_col1 + Select Operator [SEL_2] (rows=462000 width=1436) + Output:["_col0","_col1"] + Filter Operator [FIL_158] (rows=462000 width=1436) + predicate:(i_item_id is not null and i_item_sk is not null) + TableScan [TS_0] (rows=462000 width=1436) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] + <-Reducer 13 [ONE_TO_ONE_EDGE] + FORWARD [RS_102] + PartitionCols:_col0 + Group By Operator [GBY_83] (rows=115500 width=1436) + Output:["_col0"],keys:KEY._col0 + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_82] + PartitionCols:_col0 + Group By Operator [GBY_6] (rows=231000 width=1436) + Output:["_col0"],keys:i_item_id + Select Operator [SEL_5] (rows=231000 width=1436) + Output:["i_item_id"] + Filter Operator [FIL_159] (rows=231000 width=1436) + predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null) + TableScan [TS_3] (rows=462000 width=1436) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] + <-Reducer 19 [SIMPLE_EDGE] SHUFFLE [RS_67] PartitionCols:_col4 Select Operator [SEL_62] (rows=348467716 width=135) Output:["_col4","_col5"] Merge Join Operator [MERGEJOIN_178] (rows=348467716 width=135) Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 26 [SIMPLE_EDGE] + <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_60] PartitionCols:_col0 Select Operator [SEL_18] (rows=20000000 width=1014) @@ -201,12 +227,12 @@ Stage-0 predicate:((ca_gmt_offset = -8) and ca_address_sk is not null) TableScan [TS_16] (rows=40000000 width=1014) default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] + <-Reducer 18 [SIMPLE_EDGE] SHUFFLE [RS_59] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_177] (rows=316788826 width=135) Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_57] PartitionCols:_col0 Select Operator [SEL_15] (rows=18262 width=1119) @@ -215,7 +241,7 @@ Stage-0 predicate:((d_moy = 1) and (d_year = 2000) and d_date_sk is not null) TableScan [TS_13] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 27 [SIMPLE_EDGE] + <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_56] PartitionCols:_col0 Select Operator [SEL_49] (rows=287989836 width=135) @@ -224,89 +250,45 @@ Stage-0 predicate:(cs_bill_addr_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) TableScan [TS_47] (rows=287989836 width=135) default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_66] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_176] (rows=508200 width=1436) - Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_63] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=462000 width=1436) - Output:["_col0","_col1"] - Filter Operator [FIL_158] (rows=462000 width=1436) - predicate:(i_item_id is not null and i_item_sk is not null) - TableScan [TS_0] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 16 [ONE_TO_ONE_EDGE] - FORWARD [RS_64] - PartitionCols:_col0 - Group By Operator [GBY_45] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Group By Operator [GBY_6] (rows=231000 width=1436) - Output:["_col0"],keys:i_item_id - Select Operator [SEL_5] (rows=231000 width=1436) - Output:["i_item_id"] - Filter Operator [FIL_159] (rows=231000 width=1436) - predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null) - TableScan [TS_3] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"] - <-Reducer 13 [CONTAINS] + <-Reducer 4 [CONTAINS] Reduce Output Operator [RS_115] PartitionCols:_col0 Group By Operator [GBY_114] (rows=670816147 width=108) Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 Group By Operator [GBY_110] (rows=95833781 width=135) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 12 [SIMPLE_EDGE] + <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_109] PartitionCols:_col0 Group By Operator [GBY_108] (rows=191667562 width=135) Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 Merge Join Operator [MERGEJOIN_184] (rows=191667562 width=135) Conds:RS_104._col0=RS_105._col3(Inner),Output:["_col1","_col8"] - <-Reducer 11 [SIMPLE_EDGE] + <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_104] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_179] (rows=508200 width=1436) - Conds:RS_101._col1=RS_102._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_2] - <-Reducer 17 [ONE_TO_ONE_EDGE] - FORWARD [RS_102] - PartitionCols:_col0 - Group By Operator [GBY_83] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_6] - <-Reducer 25 [SIMPLE_EDGE] + Please refer to the previous Merge Join Operator [MERGEJOIN_179] + <-Reducer 21 [SIMPLE_EDGE] SHUFFLE [RS_105] PartitionCols:_col3 Select Operator [SEL_100] (rows=174243235 width=135) Output:["_col3","_col5"] Merge Join Operator [MERGEJOIN_181] (rows=174243235 width=135) Conds:RS_97._col2=RS_98._col0(Inner),Output:["_col1","_col3"] - <-Map 26 [SIMPLE_EDGE] + <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_98] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_18] - <-Reducer 24 [SIMPLE_EDGE] + <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_97] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_180] (rows=158402938 width=135) Conds:RS_94._col0=RS_95._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_95] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_15] - <-Map 28 [SIMPLE_EDGE] + <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0 Select Operator [SEL_87] (rows=144002668 width=135) @@ -315,14 +297,14 @@ Stage-0 predicate:(ws_bill_addr_sk is not null and ws_item_sk is not null and ws_sold_date_sk is not null) TableScan [TS_85] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Reducer 4 [CONTAINS] + <-Reducer 9 [CONTAINS] Reduce Output Operator [RS_115] PartitionCols:_col0 Group By Operator [GBY_114] (rows=670816147 width=108) Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 Group By Operator [GBY_35] (rows=383325119 width=88) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 3 [SIMPLE_EDGE] + <-Reducer 8 [SIMPLE_EDGE] SHUFFLE [RS_34] PartitionCols:_col0 Group By Operator [GBY_33] (rows=766650239 width=88) @@ -332,42 +314,28 @@ Stage-0 <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_29] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_173] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_2] - <-Reducer 15 [ONE_TO_ONE_EDGE] - FORWARD [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_6] - <-Reducer 20 [SIMPLE_EDGE] + Please refer to the previous Merge Join Operator [MERGEJOIN_179] + <-Reducer 16 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col3 Select Operator [SEL_25] (rows=696954748 width=88) Output:["_col3","_col5"] Merge Join Operator [MERGEJOIN_175] (rows=696954748 width=88) Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 26 [SIMPLE_EDGE] + <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_23] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_18] - <-Reducer 19 [SIMPLE_EDGE] + <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_174] (rows=633595212 width=88) Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_20] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_15] - <-Map 18 [SIMPLE_EDGE] + <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 Select Operator [SEL_12] (rows=575995635 width=88) diff --git a/ql/src/test/results/clientpositive/perf/tez/query57.q.out b/ql/src/test/results/clientpositive/perf/tez/query57.q.out index 1a1bbf1ec3..5f79e6514d 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query57.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query57.q.out @@ -95,24 +95,16 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 21 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 22 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 15 <- Map 21 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE) -Reducer 18 <- Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 3 <- Map 21 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 22 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 10 <- Reducer 5 (SIMPLE_EDGE) +Reducer 11 <- Reducer 10 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) +Reducer 3 <- Map 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 4 <- Map 14 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 13 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Reducer 7 <- Reducer 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) +Reducer 9 <- Reducer 5 (SIMPLE_EDGE) Stage-0 Fetch Operator @@ -130,78 +122,7 @@ Stage-0 Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] Merge Join Operator [MERGEJOIN_189] (rows=421645952 width=135) Conds:RS_103._col0, _col2, _col1, (_col6 + 1)=RS_104._col0, _col2, _col1, _col7(Inner),RS_104._col0, _col2, _col1, _col7=RS_105._col0, _col2, _col1, (_col6 - 1)(Inner),Output:["_col5","_col7","_col8","_col10","_col11","_col12","_col13","_col20"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col0, _col2, _col1, (_col6 + 1) - Select Operator [SEL_29] (rows=191657247 width=135) - Output:["_col0","_col1","_col2","_col5","_col6"] - Filter Operator [FIL_164] (rows=191657247 width=135) - predicate:rank_window_0 is not null - PTF Operator [PTF_28] (rows=191657247 width=135) - Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2"}] - Select Operator [SEL_27] (rows=191657247 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_25] (rows=191657247 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_24] (rows=191657247 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_23] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_22] (rows=383314495 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col10, _col11 - Merge Join Operator [MERGEJOIN_182] (rows=383314495 width=135) - Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col10","_col11"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_81] (rows=462000 width=1436) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_179] (rows=462000 width=1436) - predicate:(i_brand is not null and i_category is not null and i_item_sk is not null) - TableScan [TS_79] (rows=462000 width=1436) - default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_181] (rows=348467716 width=135) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_78] (rows=60 width=2045) - Output:["_col0","_col1"] - Filter Operator [FIL_178] (rows=60 width=2045) - predicate:(cc_call_center_sk is not null and cc_name is not null) - TableScan [TS_76] (rows=60 width=2045) - default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_name"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_180] (rows=316788826 width=135) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Select Operator [SEL_72] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_176] (rows=287989836 width=135) - predicate:(cs_call_center_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_70] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_call_center_sk","cs_item_sk","cs_sales_price"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Select Operator [SEL_75] (rows=73048 width=1119) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_177] (rows=73048 width=1119) - predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null) - TableScan [TS_73] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Reducer 19 [SIMPLE_EDGE] + <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_104] PartitionCols:_col0, _col2, _col1, _col7 Select Operator [SEL_67] (rows=15971437 width=135) @@ -216,7 +137,7 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST","partition by:":"_col1, _col2, _col3"}] Select Operator [SEL_64] (rows=191657247 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] - <-Reducer 18 [SIMPLE_EDGE] + <-Reducer 10 [SIMPLE_EDGE] SHUFFLE [RS_63] PartitionCols:_col0, _col1, _col2 Select Operator [SEL_62] (rows=191657247 width=135) @@ -225,47 +146,82 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col3 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3"}] Select Operator [SEL_60] (rows=191657247 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - <-Reducer 17 [SIMPLE_EDGE] + <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_59] PartitionCols:_col0, _col1, _col2, _col3 - Select Operator [SEL_58] (rows=191657247 width=135) + Select Operator [SEL_25] (rows=191657247 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_57] (rows=191657247 width=135) + Group By Operator [GBY_24] (rows=191657247 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_56] + <-Reducer 4 [SIMPLE_EDGE] + SHUFFLE [RS_23] PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_55] (rows=383314495 width=135) + Group By Operator [GBY_22] (rows=383314495 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col10, _col11 - Merge Join Operator [MERGEJOIN_185] (rows=383314495 width=135) - Conds:RS_51._col2=RS_52._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col10","_col11"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_52] + Merge Join Operator [MERGEJOIN_182] (rows=383314495 width=135) + Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col10","_col11"] + <-Map 14 [SIMPLE_EDGE] + SHUFFLE [RS_19] PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_81] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_51] + Select Operator [SEL_81] (rows=462000 width=1436) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_179] (rows=462000 width=1436) + predicate:(i_brand is not null and i_category is not null and i_item_sk is not null) + TableScan [TS_79] (rows=462000 width=1436) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_18] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_184] (rows=348467716 width=135) - Conds:RS_48._col1=RS_49._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_49] + Merge Join Operator [MERGEJOIN_181] (rows=348467716 width=135) + Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8"] + <-Map 13 [SIMPLE_EDGE] + SHUFFLE [RS_16] PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_78] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_48] + Select Operator [SEL_78] (rows=60 width=2045) + Output:["_col0","_col1"] + Filter Operator [FIL_178] (rows=60 width=2045) + predicate:(cc_call_center_sk is not null and cc_name is not null) + TableScan [TS_76] (rows=60 width=2045) + default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_name"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_15] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=316788826 width=135) - Conds:RS_45._col0=RS_46._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"] + Merge Join Operator [MERGEJOIN_180] (rows=316788826 width=135) + Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_45] + SHUFFLE [RS_12] PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_72] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_46] + Select Operator [SEL_72] (rows=287989836 width=135) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_176] (rows=287989836 width=135) + predicate:(cs_call_center_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) + TableScan [TS_70] (rows=287989836 width=135) + default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_call_center_sk","cs_item_sk","cs_sales_price"] + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_13] PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_75] + Select Operator [SEL_75] (rows=73048 width=1119) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_177] (rows=73048 width=1119) + predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null) + TableScan [TS_73] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] <-Reducer 6 [SIMPLE_EDGE] + SHUFFLE [RS_103] + PartitionCols:_col0, _col2, _col1, (_col6 + 1) + Select Operator [SEL_29] (rows=191657247 width=135) + Output:["_col0","_col1","_col2","_col5","_col6"] + Filter Operator [FIL_164] (rows=191657247 width=135) + predicate:rank_window_0 is not null + PTF Operator [PTF_28] (rows=191657247 width=135) + Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2"}] + Select Operator [SEL_27] (rows=191657247 width=135) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_26] + PartitionCols:_col0, _col1, _col2 + Please refer to the previous Select Operator [SEL_25] + <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_105] PartitionCols:_col0, _col2, _col1, (_col6 - 1) Select Operator [SEL_99] (rows=191657247 width=135) @@ -279,41 +235,5 @@ Stage-0 <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_96] PartitionCols:_col0, _col1, _col2 - Select Operator [SEL_95] (rows=191657247 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Group By Operator [GBY_94] (rows=191657247 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_93] - PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_92] (rows=383314495 width=135) - Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col10, _col11 - Merge Join Operator [MERGEJOIN_188] (rows=383314495 width=135) - Conds:RS_88._col2=RS_89._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col10","_col11"] - <-Map 22 [SIMPLE_EDGE] - SHUFFLE [RS_89] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_81] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_88] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_187] (rows=348467716 width=135) - Conds:RS_85._col1=RS_86._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_86] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_78] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_85] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_186] (rows=316788826 width=135) - Conds:RS_82._col0=RS_83._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_72] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_83] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_75] + Please refer to the previous Select Operator [SEL_25] diff --git a/ql/src/test/results/clientpositive/perf/tez/query58.q.out b/ql/src/test/results/clientpositive/perf/tez/query58.q.out index d5ef23bc54..769af24b4c 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query58.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query58.q.out @@ -1,6 +1,4 @@ -Warning: Shuffle Join MERGEJOIN[265][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 20' is a cross product -Warning: Shuffle Join MERGEJOIN[269][tables = [$hdt$_5, $hdt$_6]] in Stage 'Reducer 24' is a cross product -Warning: Shuffle Join MERGEJOIN[273][tables = [$hdt$_6, $hdt$_7]] in Stage 'Reducer 28' is a cross product +Warning: Shuffle Join MERGEJOIN[265][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product PREHOOK: query: explain with ss_items as (select i_item_id item_id @@ -133,31 +131,21 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 33 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) +Reducer 11 <- Map 23 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) +Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 22 (ONE_TO_ONE_EDGE) -Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 26 (ONE_TO_ONE_EDGE) -Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 30 (ONE_TO_ONE_EDGE) -Reducer 19 <- Map 18 (CUSTOM_SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 20 (ONE_TO_ONE_EDGE) +Reducer 17 <- Map 16 (CUSTOM_SIMPLE_EDGE) +Reducer 18 <- Map 21 (CUSTOM_SIMPLE_EDGE), Reducer 17 (CUSTOM_SIMPLE_EDGE) +Reducer 19 <- Map 21 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 20 <- Map 31 (CUSTOM_SIMPLE_EDGE), Reducer 19 (CUSTOM_SIMPLE_EDGE) -Reducer 21 <- Map 31 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 22 <- Reducer 21 (SIMPLE_EDGE) -Reducer 23 <- Map 18 (CUSTOM_SIMPLE_EDGE) -Reducer 24 <- Map 31 (CUSTOM_SIMPLE_EDGE), Reducer 23 (CUSTOM_SIMPLE_EDGE) -Reducer 25 <- Map 31 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) -Reducer 26 <- Reducer 25 (SIMPLE_EDGE) -Reducer 27 <- Map 18 (CUSTOM_SIMPLE_EDGE) -Reducer 28 <- Map 31 (CUSTOM_SIMPLE_EDGE), Reducer 27 (CUSTOM_SIMPLE_EDGE) -Reducer 29 <- Map 31 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE) +Reducer 20 <- Reducer 19 (SIMPLE_EDGE) Reducer 3 <- Reducer 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 32 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 16 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 8 <- Map 22 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) +Reducer 9 <- Reducer 15 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Stage-0 Fetch Operator @@ -189,13 +177,13 @@ Stage-0 Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4 Merge Join Operator [MERGEJOIN_277] (rows=696954748 width=88) Conds:RS_92._col0=RS_93._col0(Inner),Output:["_col2","_col4"] - <-Reducer 16 [SIMPLE_EDGE] + <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_93] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_271] (rows=80353 width=1119) - Conds:RS_85._col1=RS_86._col0(Inner),Output:["_col0"] + Merge Join Operator [MERGEJOIN_267] (rows=80353 width=1119) + Conds:RS_35._col1=RS_36._col0(Inner),Output:["_col0"] <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_85] + SHUFFLE [RS_35] PartitionCols:_col1 Select Operator [SEL_8] (rows=73049 width=1119) Output:["_col0","_col1"] @@ -203,48 +191,48 @@ Stage-0 predicate:(d_date is not null and d_date_sk is not null) TableScan [TS_6] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 26 [ONE_TO_ONE_EDGE] - FORWARD [RS_86] + <-Reducer 20 [ONE_TO_ONE_EDGE] + FORWARD [RS_36] PartitionCols:_col0 - Group By Operator [GBY_83] (rows=40176 width=1119) + Group By Operator [GBY_33] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_82] + <-Reducer 19 [SIMPLE_EDGE] + SHUFFLE [RS_32] PartitionCols:_col0 - Group By Operator [GBY_81] (rows=80353 width=1119) + Group By Operator [GBY_31] (rows=80353 width=1119) Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_270] (rows=80353 width=1119) - Conds:RS_77._col1=RS_78._col1(Inner),Output:["_col2"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_78] + Merge Join Operator [MERGEJOIN_266] (rows=80353 width=1119) + Conds:RS_27._col1=RS_28._col1(Inner),Output:["_col2"] + <-Map 21 [SIMPLE_EDGE] + SHUFFLE [RS_28] PartitionCols:_col1 - Select Operator [SEL_73] (rows=73049 width=1119) + Select Operator [SEL_23] (rows=73049 width=1119) Output:["_col0","_col1"] - Filter Operator [FIL_257] (rows=73049 width=1119) + Filter Operator [FIL_251] (rows=73049 width=1119) predicate:(d_date is not null and d_week_seq is not null) TableScan [TS_21] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_77] + <-Reducer 18 [SIMPLE_EDGE] + SHUFFLE [RS_27] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_269] (rows=36524 width=1128) + Merge Join Operator [MERGEJOIN_265] (rows=36524 width=1128) Conds:(Inner),Output:["_col1"] - <-Map 31 [CUSTOM_SIMPLE_EDGE] - SHUFFLE [RS_75] - Select Operator [SEL_70] (rows=36524 width=1119) + <-Map 21 [CUSTOM_SIMPLE_EDGE] + SHUFFLE [RS_25] + Select Operator [SEL_20] (rows=36524 width=1119) Output:["_col0"] - Filter Operator [FIL_256] (rows=36524 width=1119) + Filter Operator [FIL_250] (rows=36524 width=1119) predicate:((d_date = '1998-02-19') and d_week_seq is not null) Please refer to the previous TableScan [TS_21] - <-Reducer 23 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_74] - Select Operator [SEL_67] (rows=1 width=8) - Filter Operator [FIL_66] (rows=1 width=8) + <-Reducer 17 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_24] + Select Operator [SEL_17] (rows=1 width=8) + Filter Operator [FIL_16] (rows=1 width=8) predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_64] (rows=1 width=8) + Group By Operator [GBY_14] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 18 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_63] + <-Map 16 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_13] Group By Operator [GBY_12] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] Select Operator [SEL_11] (rows=36524 width=1119) @@ -266,7 +254,7 @@ Stage-0 predicate:(i_item_id is not null and i_item_sk is not null) TableScan [TS_3] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Map 32 [SIMPLE_EDGE] + <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_89] PartitionCols:_col1 Select Operator [SEL_52] (rows=575995635 width=88) @@ -287,6 +275,10 @@ Stage-0 Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4 Merge Join Operator [MERGEJOIN_278] (rows=174243235 width=135) Conds:RS_142._col0=RS_143._col0(Inner),Output:["_col2","_col4"] + <-Reducer 15 [SIMPLE_EDGE] + SHUFFLE [RS_143] + PartitionCols:_col0 + Please refer to the previous Merge Join Operator [MERGEJOIN_267] <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_142] PartitionCols:_col0 @@ -296,7 +288,7 @@ Stage-0 SHUFFLE [RS_140] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_5] - <-Map 33 [SIMPLE_EDGE] + <-Map 23 [SIMPLE_EDGE] SHUFFLE [RS_139] PartitionCols:_col1 Select Operator [SEL_102] (rows=144002668 width=135) @@ -305,57 +297,6 @@ Stage-0 predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) TableScan [TS_100] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_ext_sales_price"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_143] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_275] (rows=80353 width=1119) - Conds:RS_135._col1=RS_136._col0(Inner),Output:["_col0"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_135] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_8] - <-Reducer 30 [ONE_TO_ONE_EDGE] - FORWARD [RS_136] - PartitionCols:_col0 - Group By Operator [GBY_133] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 29 [SIMPLE_EDGE] - SHUFFLE [RS_132] - PartitionCols:_col0 - Group By Operator [GBY_131] (rows=80353 width=1119) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_274] (rows=80353 width=1119) - Conds:RS_127._col1=RS_128._col1(Inner),Output:["_col2"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_128] - PartitionCols:_col1 - Select Operator [SEL_123] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_263] (rows=73049 width=1119) - predicate:(d_date is not null and d_week_seq is not null) - Please refer to the previous TableScan [TS_21] - <-Reducer 28 [SIMPLE_EDGE] - SHUFFLE [RS_127] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_273] (rows=36524 width=1128) - Conds:(Inner),Output:["_col1"] - <-Map 31 [CUSTOM_SIMPLE_EDGE] - SHUFFLE [RS_125] - Select Operator [SEL_120] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_262] (rows=36524 width=1119) - predicate:((d_date = '1998-02-19') and d_week_seq is not null) - Please refer to the previous TableScan [TS_21] - <-Reducer 27 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_124] - Select Operator [SEL_117] (rows=1 width=8) - Filter Operator [FIL_116] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_114] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 18 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_113] - Please refer to the previous Group By Operator [GBY_12] <-Reducer 4 [ONE_TO_ONE_EDGE] FORWARD [RS_150] PartitionCols:_col0 @@ -371,54 +312,7 @@ Stage-0 <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_43] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_267] (rows=80353 width=1119) - Conds:RS_35._col1=RS_36._col0(Inner),Output:["_col0"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_8] - <-Reducer 22 [ONE_TO_ONE_EDGE] - FORWARD [RS_36] - PartitionCols:_col0 - Group By Operator [GBY_33] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_32] - PartitionCols:_col0 - Group By Operator [GBY_31] (rows=80353 width=1119) - Output:["_col0"],keys:_col2 - Merge Join Operator [MERGEJOIN_266] (rows=80353 width=1119) - Conds:RS_27._col1=RS_28._col1(Inner),Output:["_col2"] - <-Map 31 [SIMPLE_EDGE] - SHUFFLE [RS_28] - PartitionCols:_col1 - Select Operator [SEL_23] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_251] (rows=73049 width=1119) - predicate:(d_date is not null and d_week_seq is not null) - Please refer to the previous TableScan [TS_21] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_27] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_265] (rows=36524 width=1128) - Conds:(Inner),Output:["_col1"] - <-Map 31 [CUSTOM_SIMPLE_EDGE] - SHUFFLE [RS_25] - Select Operator [SEL_20] (rows=36524 width=1119) - Output:["_col0"] - Filter Operator [FIL_250] (rows=36524 width=1119) - predicate:((d_date = '1998-02-19') and d_week_seq is not null) - Please refer to the previous TableScan [TS_21] - <-Reducer 19 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_24] - Select Operator [SEL_17] (rows=1 width=8) - Filter Operator [FIL_16] (rows=1 width=8) - predicate:(sq_count_check(_col0) <= 1) - Group By Operator [GBY_14] (rows=1 width=8) - Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Map 18 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_13] - Please refer to the previous Group By Operator [GBY_12] + Please refer to the previous Merge Join Operator [MERGEJOIN_267] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_42] PartitionCols:_col0 diff --git a/ql/src/test/results/clientpositive/perf/tez/query59.q.out b/ql/src/test/results/clientpositive/perf/tez/query59.q.out index 37c2cc0147..61818a8306 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query59.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query59.q.out @@ -87,16 +87,15 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 13 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) +Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 11 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 14 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 11 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) +Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 6 <- Reducer 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (SIMPLE_EDGE) +Reducer 8 <- Reducer 2 (SIMPLE_EDGE) +Reducer 9 <- Map 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Stage-0 Fetch Operator @@ -114,14 +113,14 @@ Stage-0 Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] Merge Join Operator [MERGEJOIN_104] (rows=421657640 width=88) Conds:RS_56._col2, _col1=RS_57._col1, (_col0 - 52)(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col12","_col13","_col14","_col15","_col16","_col17"] - <-Reducer 11 [SIMPLE_EDGE] + <-Reducer 10 [SIMPLE_EDGE] SHUFFLE [RS_57] PartitionCols:_col1, (_col0 - 52) Select Operator [SEL_55] (rows=383325119 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] Merge Join Operator [MERGEJOIN_103] (rows=383325119 width=88) Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col11"] - <-Map 15 [SIMPLE_EDGE] + <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_53] PartitionCols:_col0 Select Operator [SEL_48] (rows=1704 width=1910) @@ -130,12 +129,12 @@ Stage-0 predicate:(s_store_id is not null and s_store_sk is not null) TableScan [TS_46] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id"] - <-Reducer 10 [SIMPLE_EDGE] + <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_52] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_102] (rows=348477374 width=88) Conds:RS_49._col0=RS_50._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] - <-Map 13 [SIMPLE_EDGE] + <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_50] PartitionCols:_col1 Select Operator [SEL_45] (rows=8116 width=1119) @@ -144,22 +143,22 @@ Stage-0 predicate:(d_month_seq BETWEEN 1197 AND 1208 and d_week_seq is not null) TableScan [TS_15] (rows=73049 width=1119) default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_week_seq"] - <-Reducer 9 [SIMPLE_EDGE] + <-Reducer 8 [SIMPLE_EDGE] SHUFFLE [RS_49] PartitionCols:_col0 Group By Operator [GBY_41] (rows=316797606 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)"],keys:KEY._col0, KEY._col1 - <-Reducer 8 [SIMPLE_EDGE] + <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_40] PartitionCols:_col0, _col1 Group By Operator [GBY_39] (rows=633595212 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col2)","sum(_col3)","sum(_col5)","sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col0, _col1 Select Operator [SEL_37] (rows=633595212 width=88) Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col7","_col8"] - Merge Join Operator [MERGEJOIN_101] (rows=633595212 width=88) - Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col1","_col2","_col4","_col5"] + Merge Join Operator [MERGEJOIN_98] (rows=633595212 width=88) + Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col4","_col5"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_34] + SHUFFLE [RS_6] PartitionCols:_col0 Select Operator [SEL_2] (rows=575995635 width=88) Output:["_col0","_col1","_col2"] @@ -167,8 +166,8 @@ Stage-0 predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_sales_price"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_35] + <-Map 11 [SIMPLE_EDGE] + SHUFFLE [RS_7] PartitionCols:_col0 Select Operator [SEL_5] (rows=73049 width=1119) Output:["_col0","_col1","_col2"] @@ -183,7 +182,7 @@ Stage-0 Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] Merge Join Operator [MERGEJOIN_100] (rows=383325119 width=88) Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col12","_col13"] - <-Map 14 [SIMPLE_EDGE] + <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_25] PartitionCols:_col0 Select Operator [SEL_20] (rows=1704 width=1910) @@ -197,7 +196,7 @@ Stage-0 PartitionCols:_col1 Merge Join Operator [MERGEJOIN_99] (rows=348477374 width=88) Conds:RS_21._col0=RS_22._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - <-Map 13 [SIMPLE_EDGE] + <-Map 12 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col1 Select Operator [SEL_17] (rows=8116 width=1119) @@ -217,14 +216,5 @@ Stage-0 Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)","sum(_col5)","sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col0, _col1 Select Operator [SEL_9] (rows=633595212 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Merge Join Operator [MERGEJOIN_98] (rows=633595212 width=88) - Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col4","_col5"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_6] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_2] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_5] + Please refer to the previous Merge Join Operator [MERGEJOIN_98] diff --git a/ql/src/test/results/clientpositive/perf/tez/query60.q.out b/ql/src/test/results/clientpositive/perf/tez/query60.q.out index 84a9ada7e8..67102b377b 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query60.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query60.q.out @@ -155,26 +155,22 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (ONE_TO_ONE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 15 <- Map 14 (SIMPLE_EDGE) -Reducer 16 <- Map 14 (SIMPLE_EDGE) -Reducer 17 <- Map 14 (SIMPLE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE) -Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) -Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE) -Reducer 23 <- Map 26 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 24 <- Map 21 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE) -Reducer 25 <- Map 26 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) +Reducer 10 <- Reducer 2 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) +Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Union 5 (CONTAINS) +Reducer 13 <- Map 12 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) +Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) +Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) +Reducer 19 <- Map 22 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE) +Reducer 20 <- Map 17 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) +Reducer 21 <- Map 22 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) +Reducer 3 <- Reducer 16 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (ONE_TO_ONE_EDGE) -Reducer 9 <- Reducer 23 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 8 <- Reducer 19 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 9 <- Reducer 8 (SIMPLE_EDGE), Union 5 (CONTAINS) Stage-0 Fetch Operator @@ -191,66 +187,27 @@ Stage-0 Group By Operator [GBY_116] (rows=335408073 width=108) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 <-Union 5 [SIMPLE_EDGE] - <-Reducer 10 [CONTAINS] + <-Reducer 11 [CONTAINS] Reduce Output Operator [RS_115] PartitionCols:_col0 Group By Operator [GBY_114] (rows=670816147 width=108) Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_72] (rows=191657247 width=135) + Group By Operator [GBY_110] (rows=95833781 width=135) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_71] + <-Reducer 10 [SIMPLE_EDGE] + SHUFFLE [RS_109] PartitionCols:_col0 - Group By Operator [GBY_70] (rows=383314495 width=135) + Group By Operator [GBY_108] (rows=191667562 width=135) Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135) - Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_67] - PartitionCols:_col4 - Select Operator [SEL_62] (rows=348467716 width=135) - Output:["_col4","_col5"] - Merge Join Operator [MERGEJOIN_178] (rows=348467716 width=135) - Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] - <-Map 26 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0 - Select Operator [SEL_18] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_162] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) - TableScan [TS_16] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_177] (rows=316788826 width=135) - Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] - SHUFFLE [RS_57] - PartitionCols:_col0 - Select Operator [SEL_15] (rows=18262 width=1119) - Output:["_col0"] - Filter Operator [FIL_161] (rows=18262 width=1119) - predicate:((d_moy = 9) and (d_year = 1999) and d_date_sk is not null) - TableScan [TS_13] (rows=73049 width=1119) - default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 27 [SIMPLE_EDGE] - SHUFFLE [RS_56] - PartitionCols:_col0 - Select Operator [SEL_49] (rows=287989836 width=135) - Output:["_col0","_col1","_col2","_col3"] - Filter Operator [FIL_165] (rows=287989836 width=135) - predicate:(cs_bill_addr_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) - TableScan [TS_47] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_66] + Merge Join Operator [MERGEJOIN_184] (rows=191667562 width=135) + Conds:RS_104._col0=RS_105._col3(Inner),Output:["_col1","_col8"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_104] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_176] (rows=508200 width=1436) - Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"] + Merge Join Operator [MERGEJOIN_173] (rows=508200 width=1436) + Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_63] + SHUFFLE [RS_26] PartitionCols:_col1 Select Operator [SEL_2] (rows=462000 width=1436) Output:["_col0","_col1"] @@ -258,13 +215,13 @@ Stage-0 predicate:(i_item_id is not null and i_item_sk is not null) TableScan [TS_0] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 16 [ONE_TO_ONE_EDGE] - FORWARD [RS_64] + <-Reducer 13 [ONE_TO_ONE_EDGE] + FORWARD [RS_27] PartitionCols:_col0 - Group By Operator [GBY_45] (rows=115500 width=1436) + Group By Operator [GBY_8] (rows=115500 width=1436) Output:["_col0"],keys:KEY._col0 - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_44] + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_7] PartitionCols:_col0 Group By Operator [GBY_6] (rows=231000 width=1436) Output:["_col0"],keys:i_item_id @@ -274,59 +231,37 @@ Stage-0 predicate:((i_category) IN ('Children') and i_item_id is not null) TableScan [TS_3] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"] - <-Reducer 13 [CONTAINS] - Reduce Output Operator [RS_115] - PartitionCols:_col0 - Group By Operator [GBY_114] (rows=670816147 width=108) - Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 - Group By Operator [GBY_110] (rows=95833781 width=135) - Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col0 - Group By Operator [GBY_108] (rows=191667562 width=135) - Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 - Merge Join Operator [MERGEJOIN_184] (rows=191667562 width=135) - Conds:RS_104._col0=RS_105._col3(Inner),Output:["_col1","_col8"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_179] (rows=508200 width=1436) - Conds:RS_101._col1=RS_102._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_2] - <-Reducer 17 [ONE_TO_ONE_EDGE] - FORWARD [RS_102] - PartitionCols:_col0 - Group By Operator [GBY_83] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_82] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_6] - <-Reducer 25 [SIMPLE_EDGE] + <-Reducer 21 [SIMPLE_EDGE] SHUFFLE [RS_105] PartitionCols:_col3 Select Operator [SEL_100] (rows=174243235 width=135) Output:["_col3","_col5"] Merge Join Operator [MERGEJOIN_181] (rows=174243235 width=135) Conds:RS_97._col2=RS_98._col0(Inner),Output:["_col1","_col3"] - <-Map 26 [SIMPLE_EDGE] + <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_98] PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_18] - <-Reducer 24 [SIMPLE_EDGE] + Select Operator [SEL_18] (rows=20000000 width=1014) + Output:["_col0"] + Filter Operator [FIL_162] (rows=20000000 width=1014) + predicate:((ca_gmt_offset = -6) and ca_address_sk is not null) + TableScan [TS_16] (rows=40000000 width=1014) + default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] + <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_97] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_180] (rows=158402938 width=135) Conds:RS_94._col0=RS_95._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_95] PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_15] - <-Map 28 [SIMPLE_EDGE] + Select Operator [SEL_15] (rows=18262 width=1119) + Output:["_col0"] + Filter Operator [FIL_161] (rows=18262 width=1119) + predicate:((d_moy = 9) and (d_year = 1999) and d_date_sk is not null) + TableScan [TS_13] (rows=73049 width=1119) + default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + <-Map 24 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0 Select Operator [SEL_87] (rows=144002668 width=135) @@ -352,42 +287,28 @@ Stage-0 <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_29] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_173] (rows=508200 width=1436) - Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_26] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_2] - <-Reducer 15 [ONE_TO_ONE_EDGE] - FORWARD [RS_27] - PartitionCols:_col0 - Group By Operator [GBY_8] (rows=115500 width=1436) - Output:["_col0"],keys:KEY._col0 - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_7] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_6] - <-Reducer 20 [SIMPLE_EDGE] + Please refer to the previous Merge Join Operator [MERGEJOIN_173] + <-Reducer 16 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col3 Select Operator [SEL_25] (rows=696954748 width=88) Output:["_col3","_col5"] Merge Join Operator [MERGEJOIN_175] (rows=696954748 width=88) Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"] - <-Map 26 [SIMPLE_EDGE] + <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_23] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_18] - <-Reducer 19 [SIMPLE_EDGE] + <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_174] (rows=633595212 width=88) Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"] - <-Map 21 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_20] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_15] - <-Map 18 [SIMPLE_EDGE] + <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_19] PartitionCols:_col0 Select Operator [SEL_12] (rows=575995635 width=88) @@ -396,4 +317,51 @@ Stage-0 predicate:(ss_addr_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null) TableScan [TS_10] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"] + <-Reducer 9 [CONTAINS] + Reduce Output Operator [RS_115] + PartitionCols:_col0 + Group By Operator [GBY_114] (rows=670816147 width=108) + Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0 + Group By Operator [GBY_72] (rows=191657247 width=135) + Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 + <-Reducer 8 [SIMPLE_EDGE] + SHUFFLE [RS_71] + PartitionCols:_col0 + Group By Operator [GBY_70] (rows=383314495 width=135) + Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1 + Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135) + Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_66] + PartitionCols:_col0 + Please refer to the previous Merge Join Operator [MERGEJOIN_173] + <-Reducer 19 [SIMPLE_EDGE] + SHUFFLE [RS_67] + PartitionCols:_col4 + Select Operator [SEL_62] (rows=348467716 width=135) + Output:["_col4","_col5"] + Merge Join Operator [MERGEJOIN_178] (rows=348467716 width=135) + Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"] + <-Map 22 [SIMPLE_EDGE] + SHUFFLE [RS_60] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_18] + <-Reducer 18 [SIMPLE_EDGE] + SHUFFLE [RS_59] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_177] (rows=316788826 width=135) + Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"] + <-Map 17 [SIMPLE_EDGE] + SHUFFLE [RS_57] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_15] + <-Map 23 [SIMPLE_EDGE] + SHUFFLE [RS_56] + PartitionCols:_col0 + Select Operator [SEL_49] (rows=287989836 width=135) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_165] (rows=287989836 width=135) + predicate:(cs_bill_addr_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) + TableScan [TS_47] (rows=287989836 width=135) + default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"] diff --git a/ql/src/test/results/clientpositive/perf/tez/query61.q.out b/ql/src/test/results/clientpositive/perf/tez/query61.q.out index 0b4f5fd400..7ac7c0ffda 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query61.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query61.q.out @@ -88,21 +88,20 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) +Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE) +Reducer 12 <- Map 19 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) Reducer 13 <- Map 20 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) Reducer 14 <- Map 21 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) -Reducer 15 <- Map 22 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE) +Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 22 (SIMPLE_EDGE) +Reducer 17 <- Map 19 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) Reducer 18 <- Map 20 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Map 21 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE) -Reducer 3 <- Reducer 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) +Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) -Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE), Reducer 9 (CUSTOM_SIMPLE_EDGE) +Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE) -Reducer 8 <- Reducer 19 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 8 (CUSTOM_SIMPLE_EDGE) +Reducer 7 <- Reducer 18 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE) Stage-0 Fetch Operator @@ -130,12 +129,35 @@ Stage-0 Output:["_col0"],aggregations:["sum(_col9)"] Merge Join Operator [MERGEJOIN_154] (rows=927646829 width=88) Conds:RS_37._col0=RS_38._col2(Inner),Output:["_col9"] - <-Reducer 15 [SIMPLE_EDGE] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_37] + PartitionCols:_col0 + Merge Join Operator [MERGEJOIN_145] (rows=88000001 width=860) + Conds:RS_34._col1=RS_35._col0(Inner),Output:["_col0"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_34] + PartitionCols:_col1 + Select Operator [SEL_2] (rows=80000000 width=860) + Output:["_col0","_col1"] + Filter Operator [FIL_132] (rows=80000000 width=860) + predicate:(c_current_addr_sk is not null and c_customer_sk is not null) + TableScan [TS_0] (rows=80000000 width=860) + default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] + <-Map 9 [SIMPLE_EDGE] + SHUFFLE [RS_35] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=20000000 width=1014) + Output:["_col0"] + Filter Operator [FIL_133] (rows=20000000 width=1014) + predicate:((ca_gmt_offset = -7) and ca_address_sk is not null) + TableScan [TS_3] (rows=40000000 width=1014) + default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] + <-Reducer 14 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_149] (rows=843315281 width=88) Conds:RS_30._col4=RS_31._col0(Inner),Output:["_col2","_col5"] - <-Map 22 [SIMPLE_EDGE] + <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_31] PartitionCols:_col0 Select Operator [SEL_20] (rows=2300 width=1179) @@ -144,12 +166,12 @@ Stage-0 predicate:(((p_channel_dmail = 'Y') or (p_channel_email = 'Y') or (p_channel_tv = 'Y')) and p_promo_sk is not null) TableScan [TS_18] (rows=2300 width=1179) default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_dmail","p_channel_email","p_channel_tv"] - <-Reducer 14 [SIMPLE_EDGE] + <-Reducer 13 [SIMPLE_EDGE] SHUFFLE [RS_30] PartitionCols:_col4 Merge Join Operator [MERGEJOIN_148] (rows=766650239 width=88) Conds:RS_27._col3=RS_28._col0(Inner),Output:["_col2","_col4","_col5"] - <-Map 21 [SIMPLE_EDGE] + <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_28] PartitionCols:_col0 Select Operator [SEL_17] (rows=852 width=1910) @@ -158,12 +180,12 @@ Stage-0 predicate:((s_gmt_offset = -7) and s_store_sk is not null) TableScan [TS_15] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_gmt_offset"] - <-Reducer 13 [SIMPLE_EDGE] + <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_27] PartitionCols:_col3 Merge Join Operator [MERGEJOIN_147] (rows=696954748 width=88) Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col2","_col3","_col4","_col5"] - <-Map 20 [SIMPLE_EDGE] + <-Map 19 [SIMPLE_EDGE] SHUFFLE [RS_25] PartitionCols:_col0 Select Operator [SEL_14] (rows=231000 width=1436) @@ -172,12 +194,12 @@ Stage-0 predicate:((i_category = 'Electronics') and i_item_sk is not null) TableScan [TS_12] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"] - <-Reducer 12 [SIMPLE_EDGE] + <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_24] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_146] (rows=633595212 width=88) Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"] - <-Map 16 [SIMPLE_EDGE] + <-Map 15 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col0 Select Operator [SEL_11] (rows=18262 width=1119) @@ -186,7 +208,7 @@ Stage-0 predicate:((d_moy = 11) and (d_year = 1999) and d_date_sk is not null) TableScan [TS_9] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] - <-Map 11 [SIMPLE_EDGE] + <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_21] PartitionCols:_col0 Select Operator [SEL_8] (rows=575995635 width=88) @@ -195,67 +217,48 @@ Stage-0 predicate:(ss_customer_sk is not null and ss_item_sk is not null and ss_promo_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null) TableScan [TS_6] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_promo_sk","ss_ext_sales_price"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_145] (rows=88000001 width=860) - Conds:RS_34._col1=RS_35._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_34] - PartitionCols:_col1 - Select Operator [SEL_2] (rows=80000000 width=860) - Output:["_col0","_col1"] - Filter Operator [FIL_132] (rows=80000000 width=860) - predicate:(c_current_addr_sk is not null and c_customer_sk is not null) - TableScan [TS_0] (rows=80000000 width=860) - default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_35] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=20000000 width=1014) - Output:["_col0"] - Filter Operator [FIL_133] (rows=20000000 width=1014) - predicate:((ca_gmt_offset = -7) and ca_address_sk is not null) - TableScan [TS_3] (rows=40000000 width=1014) - default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"] - <-Reducer 9 [CUSTOM_SIMPLE_EDGE] + <-Reducer 8 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_85] Group By Operator [GBY_82] (rows=1 width=112) Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Reducer 8 [CUSTOM_SIMPLE_EDGE] + <-Reducer 7 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_81] Group By Operator [GBY_80] (rows=1 width=112) Output:["_col0"],aggregations:["sum(_col8)"] Merge Join Operator [MERGEJOIN_155] (rows=843315281 width=88) Conds:RS_76._col0=RS_77._col2(Inner),Output:["_col8"] - <-Reducer 19 [SIMPLE_EDGE] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_76] + PartitionCols:_col0 + Please refer to the previous Merge Join Operator [MERGEJOIN_145] + <-Reducer 18 [SIMPLE_EDGE] SHUFFLE [RS_77] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_153] (rows=766650239 width=88) Conds:RS_69._col3=RS_70._col0(Inner),Output:["_col2","_col4"] - <-Map 21 [SIMPLE_EDGE] + <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_70] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_17] - <-Reducer 18 [SIMPLE_EDGE] + <-Reducer 17 [SIMPLE_EDGE] SHUFFLE [RS_69] PartitionCols:_col3 Merge Join Operator [MERGEJOIN_152] (rows=696954748 width=88) Conds:RS_66._col1=RS_67._col0(Inner),Output:["_col2","_col3","_col4"] - <-Map 20 [SIMPLE_EDGE] + <-Map 19 [SIMPLE_EDGE] SHUFFLE [RS_67] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_14] - <-Reducer 17 [SIMPLE_EDGE] + <-Reducer 16 [SIMPLE_EDGE] SHUFFLE [RS_66] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_151] (rows=633595212 width=88) Conds:RS_63._col0=RS_64._col0(Inner),Output:["_col1","_col2","_col3","_col4"] - <-Map 16 [SIMPLE_EDGE] + <-Map 15 [SIMPLE_EDGE] SHUFFLE [RS_64] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_11] - <-Map 23 [SIMPLE_EDGE] + <-Map 22 [SIMPLE_EDGE] SHUFFLE [RS_63] PartitionCols:_col0 Select Operator [SEL_53] (rows=575995635 width=88) @@ -264,17 +267,4 @@ Stage-0 predicate:(ss_customer_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null) TableScan [TS_51] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ext_sales_price"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_76] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_150] (rows=88000001 width=860) - Conds:RS_73._col1=RS_74._col0(Inner),Output:["_col0"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_73] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_2] - <-Map 10 [SIMPLE_EDGE] - SHUFFLE [RS_74] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_5] diff --git a/ql/src/test/results/clientpositive/perf/tez/query64.q.out b/ql/src/test/results/clientpositive/perf/tez/query64.q.out index e009eaf66b..ecc761f34c 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query64.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query64.q.out @@ -237,47 +237,36 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 19 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) +Reducer 10 <- Reducer 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 13 <- Map 20 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) -Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE) -Reducer 15 <- Map 41 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Map 55 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) -Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE) -Reducer 18 <- Map 55 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) -Reducer 19 <- Reducer 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) -Reducer 21 <- Map 20 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) -Reducer 22 <- Map 46 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) -Reducer 23 <- Reducer 22 (SIMPLE_EDGE), Reducer 37 (SIMPLE_EDGE) -Reducer 24 <- Reducer 23 (SIMPLE_EDGE), Reducer 49 (ONE_TO_ONE_EDGE) -Reducer 25 <- Map 53 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) -Reducer 26 <- Map 41 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) -Reducer 27 <- Map 54 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 28 <- Map 20 (SIMPLE_EDGE), Reducer 44 (SIMPLE_EDGE) -Reducer 29 <- Map 46 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE) -Reducer 3 <- Map 20 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (SIMPLE_EDGE), Reducer 39 (SIMPLE_EDGE) -Reducer 31 <- Reducer 30 (SIMPLE_EDGE), Reducer 51 (ONE_TO_ONE_EDGE) -Reducer 32 <- Map 53 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) -Reducer 33 <- Map 41 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE) -Reducer 34 <- Map 54 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE) -Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE) -Reducer 37 <- Map 35 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE) -Reducer 38 <- Map 35 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE) -Reducer 39 <- Map 35 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE) -Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 45 (SIMPLE_EDGE) -Reducer 44 <- Map 42 (SIMPLE_EDGE), Map 45 (SIMPLE_EDGE) -Reducer 48 <- Map 47 (SIMPLE_EDGE), Map 52 (SIMPLE_EDGE) -Reducer 49 <- Reducer 48 (SIMPLE_EDGE) -Reducer 5 <- Map 41 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 50 <- Map 47 (SIMPLE_EDGE), Map 52 (SIMPLE_EDGE) -Reducer 51 <- Reducer 50 (SIMPLE_EDGE) -Reducer 6 <- Map 55 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Reducer 27 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 55 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Reducer 12 <- Reducer 29 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Reducer 13 <- Map 44 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) +Reducer 14 <- Reducer 13 (SIMPLE_EDGE) +Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 35 (SIMPLE_EDGE) +Reducer 17 <- Map 37 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) +Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) +Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 40 (ONE_TO_ONE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE) +Reducer 20 <- Map 42 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) +Reducer 21 <- Map 33 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) +Reducer 22 <- Map 43 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) +Reducer 23 <- Map 15 (SIMPLE_EDGE), Reducer 35 (SIMPLE_EDGE) +Reducer 24 <- Map 37 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE) +Reducer 25 <- Reducer 24 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) +Reducer 26 <- Reducer 25 (SIMPLE_EDGE), Reducer 40 (ONE_TO_ONE_EDGE) +Reducer 27 <- Map 42 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) +Reducer 28 <- Map 33 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) +Reducer 29 <- Map 43 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE) +Reducer 3 <- Map 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 31 <- Map 30 (SIMPLE_EDGE), Map 32 (SIMPLE_EDGE) +Reducer 35 <- Map 34 (SIMPLE_EDGE), Map 36 (SIMPLE_EDGE) +Reducer 39 <- Map 38 (SIMPLE_EDGE), Map 41 (SIMPLE_EDGE) +Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) +Reducer 40 <- Reducer 39 (SIMPLE_EDGE) +Reducer 5 <- Map 33 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 6 <- Map 44 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) +Reducer 7 <- Reducer 22 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Reducer 8 <- Map 44 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE) Stage-0 @@ -296,14 +285,14 @@ Stage-0 predicate:(_col30 <= _col13) Merge Join Operator [MERGEJOIN_615] (rows=821691577 width=88) Conds:RS_256._col2, _col1, _col3=RS_257._col2, _col1, _col3(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col13","_col14","_col15","_col16","_col30","_col31","_col32","_col33"] - <-Reducer 19 [SIMPLE_EDGE] + <-Reducer 14 [SIMPLE_EDGE] SHUFFLE [RS_257] PartitionCols:_col2, _col1, _col3 Select Operator [SEL_254] (rows=746992327 width=88) Output:["_col1","_col2","_col3","_col13","_col14","_col15","_col16"] Group By Operator [GBY_253] (rows=746992327 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13 - <-Reducer 18 [SIMPLE_EDGE] + <-Reducer 13 [SIMPLE_EDGE] SHUFFLE [RS_252] PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 Group By Operator [GBY_251] (rows=1493984654 width=88) @@ -314,7 +303,7 @@ Stage-0 predicate:(_col56 <> _col19) Merge Join Operator [MERGEJOIN_614] (rows=1493984654 width=88) Conds:RS_246._col37=RS_247._col0(Inner),Output:["_col7","_col9","_col14","_col15","_col16","_col17","_col19","_col23","_col24","_col25","_col26","_col28","_col29","_col43","_col44","_col45","_col46","_col49","_col56"] - <-Map 55 [SIMPLE_EDGE] + <-Map 44 [SIMPLE_EDGE] SHUFFLE [RS_247] PartitionCols:_col0 Select Operator [SEL_99] (rows=1861800 width=385) @@ -323,26 +312,26 @@ Stage-0 predicate:cd_demo_sk is not null TableScan [TS_97] (rows=1861800 width=385) default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"] - <-Reducer 17 [SIMPLE_EDGE] + <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_246] PartitionCols:_col37 Merge Join Operator [MERGEJOIN_613] (rows=1358167838 width=88) Conds:RS_243._col0=RS_244._col16(Inner),Output:["_col7","_col9","_col14","_col15","_col16","_col17","_col19","_col23","_col24","_col25","_col26","_col28","_col29","_col37","_col43","_col44","_col45","_col46","_col49"] - <-Reducer 16 [SIMPLE_EDGE] + <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_243] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_602] (rows=128840811 width=860) - Conds:RS_240._col1=RS_241._col0(Inner),Output:["_col0","_col7","_col9","_col14","_col15","_col16","_col17","_col19"] - <-Map 55 [SIMPLE_EDGE] - SHUFFLE [RS_241] + Merge Join Operator [MERGEJOIN_584] (rows=128840811 width=860) + Conds:RS_112._col1=RS_113._col0(Inner),Output:["_col0","_col7","_col9","_col14","_col15","_col16","_col17","_col19"] + <-Map 44 [SIMPLE_EDGE] + SHUFFLE [RS_113] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_99] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_240] + <-Reducer 5 [SIMPLE_EDGE] + SHUFFLE [RS_112] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_601] (rows=117128008 width=860) Conds:RS_237._col3=RS_238._col0(Inner),Output:["_col0","_col1","_col7","_col9","_col14","_col15","_col16","_col17"] - <-Map 41 [SIMPLE_EDGE] + <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_238] PartitionCols:_col0 Select Operator [SEL_21] (rows=40000000 width=1014) @@ -351,17 +340,40 @@ Stage-0 predicate:ca_address_sk is not null TableScan [TS_19] (rows=40000000 width=1014) default@customer_address,ad2,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"] - <-Reducer 14 [SIMPLE_EDGE] + <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_237] PartitionCols:_col3 Merge Join Operator [MERGEJOIN_600] (rows=106480005 width=860) Conds:RS_234._col2=RS_235._col0(Inner),Output:["_col0","_col1","_col3","_col7","_col9"] - <-Reducer 13 [SIMPLE_EDGE] + <-Reducer 31 [SIMPLE_EDGE] + SHUFFLE [RS_235] + PartitionCols:_col0 + Merge Join Operator [MERGEJOIN_603] (rows=7920 width=107) + Conds:RS_206._col1=RS_207._col0(Inner),Output:["_col0"] + <-Map 30 [SIMPLE_EDGE] + SHUFFLE [RS_206] + PartitionCols:_col1 + Select Operator [SEL_11] (rows=7200 width=107) + Output:["_col0","_col1"] + Filter Operator [FIL_544] (rows=7200 width=107) + predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) + TableScan [TS_9] (rows=7200 width=107) + default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] + <-Map 32 [SIMPLE_EDGE] + SHUFFLE [RS_207] + PartitionCols:_col0 + Select Operator [SEL_14] (rows=20 width=12) + Output:["_col0"] + Filter Operator [FIL_545] (rows=20 width=12) + predicate:ib_income_band_sk is not null + TableScan [TS_12] (rows=20 width=12) + default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] + <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_234] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_598] (rows=96800003 width=860) Conds:RS_231._col4=RS_232._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col7","_col9"] - <-Map 20 [SIMPLE_EDGE] + <-Map 15 [SIMPLE_EDGE] SHUFFLE [RS_232] PartitionCols:_col0 Select Operator [SEL_136] (rows=73049 width=1119) @@ -370,12 +382,12 @@ Stage-0 predicate:d_date_sk is not null TableScan [TS_3] (rows=73049 width=1119) default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"] - <-Reducer 12 [SIMPLE_EDGE] + <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_231] PartitionCols:_col4 Merge Join Operator [MERGEJOIN_597] (rows=88000001 width=860) Conds:RS_228._col5=RS_229._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7"] - <-Map 20 [SIMPLE_EDGE] + <-Map 15 [SIMPLE_EDGE] SHUFFLE [RS_229] PartitionCols:_col0 Select Operator [SEL_133] (rows=73049 width=1119) @@ -392,37 +404,14 @@ Stage-0 predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_customer_sk is not null and c_first_sales_date_sk is not null and c_first_shipto_date_sk is not null) TableScan [TS_0] (rows=80000000 width=860) default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_shipto_date_sk","c_first_sales_date_sk"] - <-Reducer 38 [SIMPLE_EDGE] - SHUFFLE [RS_235] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_599] (rows=7920 width=107) - Conds:RS_143._col1=RS_144._col0(Inner),Output:["_col0"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_143] - PartitionCols:_col1 - Select Operator [SEL_11] (rows=7200 width=107) - Output:["_col0","_col1"] - Filter Operator [FIL_544] (rows=7200 width=107) - predicate:(hd_demo_sk is not null and hd_income_band_sk is not null) - TableScan [TS_9] (rows=7200 width=107) - default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_144] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=20 width=12) - Output:["_col0"] - Filter Operator [FIL_545] (rows=20 width=12) - predicate:ib_income_band_sk is not null - TableScan [TS_12] (rows=20 width=12) - default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"] - <-Reducer 34 [SIMPLE_EDGE] + <-Reducer 29 [SIMPLE_EDGE] SHUFFLE [RS_244] PartitionCols:_col16 Select Operator [SEL_224] (rows=1234698008 width=88) Output:["_col3","_col4","_col5","_col6","_col8","_col9","_col16","_col17","_col23","_col24","_col25","_col26","_col29"] Merge Join Operator [MERGEJOIN_612] (rows=1234698008 width=88) Conds:RS_221._col5, _col12=RS_222._col0, _col1(Inner),Output:["_col6","_col7","_col13","_col14","_col15","_col16","_col19","_col26","_col27","_col29","_col30","_col31","_col32"] - <-Map 54 [SIMPLE_EDGE] + <-Map 43 [SIMPLE_EDGE] SHUFFLE [RS_222] PartitionCols:_col0, _col1 Select Operator [SEL_77] (rows=57591150 width=77) @@ -431,21 +420,21 @@ Stage-0 predicate:(sr_item_sk is not null and sr_ticket_number is not null) TableScan [TS_75] (rows=57591150 width=77) default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] - <-Reducer 33 [SIMPLE_EDGE] + <-Reducer 28 [SIMPLE_EDGE] SHUFFLE [RS_221] PartitionCols:_col5, _col12 Merge Join Operator [MERGEJOIN_611] (rows=1122452711 width=88) Conds:RS_218._col9=RS_219._col0(Inner),Output:["_col5","_col6","_col7","_col12","_col13","_col14","_col15","_col16","_col19","_col26","_col27","_col29","_col30","_col31","_col32"] - <-Map 41 [SIMPLE_EDGE] + <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_219] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_21] - <-Reducer 32 [SIMPLE_EDGE] + <-Reducer 27 [SIMPLE_EDGE] SHUFFLE [RS_218] PartitionCols:_col9 Merge Join Operator [MERGEJOIN_610] (rows=1020411534 width=88) Conds:RS_215._col10=RS_216._col0(Inner),Output:["_col5","_col6","_col7","_col9","_col12","_col13","_col14","_col15","_col16","_col19","_col26","_col27"] - <-Map 53 [SIMPLE_EDGE] + <-Map 42 [SIMPLE_EDGE] SHUFFLE [RS_216] PartitionCols:_col0 Select Operator [SEL_71] (rows=1704 width=1910) @@ -454,24 +443,64 @@ Stage-0 predicate:(s_store_name is not null and s_store_sk is not null and s_zip is not null) TableScan [TS_69] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] - <-Reducer 31 [SIMPLE_EDGE] + <-Reducer 26 [SIMPLE_EDGE] SHUFFLE [RS_215] PartitionCols:_col10 Merge Join Operator [MERGEJOIN_609] (rows=927646829 width=88) Conds:RS_212._col5=RS_213._col0(Inner),Output:["_col5","_col6","_col7","_col9","_col10","_col12","_col13","_col14","_col15","_col16","_col19"] - <-Reducer 30 [SIMPLE_EDGE] + <-Reducer 40 [ONE_TO_ONE_EDGE] + FORWARD [RS_213] + PartitionCols:_col0 + Select Operator [SEL_196] (rows=52798137 width=135) + Output:["_col0"] + Filter Operator [FIL_195] (rows=52798137 width=135) + predicate:(_col1 > (2 * _col2)) + Group By Operator [GBY_194] (rows=158394413 width=135) + Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 + <-Reducer 39 [SIMPLE_EDGE] + SHUFFLE [RS_193] + PartitionCols:_col0 + Group By Operator [GBY_64] (rows=316788826 width=135) + Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0 + Select Operator [SEL_62] (rows=316788826 width=135) + Output:["_col0","_col1","_col2"] + Merge Join Operator [MERGEJOIN_589] (rows=316788826 width=135) + Conds:RS_59._col0, _col1=RS_60._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"] + <-Map 38 [SIMPLE_EDGE] + SHUFFLE [RS_59] + PartitionCols:_col0, _col1 + Select Operator [SEL_55] (rows=287989836 width=135) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_554] (rows=287989836 width=135) + predicate:(cs_item_sk is not null and cs_order_number is not null) + TableScan [TS_53] (rows=287989836 width=135) + default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"] + <-Map 41 [SIMPLE_EDGE] + SHUFFLE [RS_60] + PartitionCols:_col0, _col1 + Select Operator [SEL_58] (rows=28798881 width=106) + Output:["_col0","_col1","_col2","_col3","_col4"] + Filter Operator [FIL_555] (rows=28798881 width=106) + predicate:(cr_item_sk is not null and cr_order_number is not null) + TableScan [TS_56] (rows=28798881 width=106) + default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"] + <-Reducer 25 [SIMPLE_EDGE] SHUFFLE [RS_212] PartitionCols:_col5 Merge Join Operator [MERGEJOIN_608] (rows=843315281 width=88) Conds:RS_209._col0=RS_210._col5(Inner),Output:["_col5","_col6","_col7","_col9","_col10","_col12","_col13","_col14","_col15","_col16","_col19"] - <-Reducer 29 [SIMPLE_EDGE] + <-Reducer 31 [SIMPLE_EDGE] + SHUFFLE [RS_209] + PartitionCols:_col0 + Please refer to the previous Merge Join Operator [MERGEJOIN_603] + <-Reducer 24 [SIMPLE_EDGE] SHUFFLE [RS_210] PartitionCols:_col5 Select Operator [SEL_180] (rows=766650239 width=88) Output:["_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col13","_col16"] Merge Join Operator [MERGEJOIN_606] (rows=766650239 width=88) Conds:RS_177._col7=RS_178._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col8","_col9","_col10","_col11","_col12","_col15"] - <-Map 46 [SIMPLE_EDGE] + <-Map 37 [SIMPLE_EDGE] SHUFFLE [RS_178] PartitionCols:_col0 Select Operator [SEL_42] (rows=2300 width=1179) @@ -480,12 +509,12 @@ Stage-0 predicate:p_promo_sk is not null TableScan [TS_40] (rows=2300 width=1179) default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"] - <-Reducer 28 [SIMPLE_EDGE] + <-Reducer 23 [SIMPLE_EDGE] SHUFFLE [RS_177] PartitionCols:_col7 Merge Join Operator [MERGEJOIN_605] (rows=696954748 width=88) Conds:RS_174._col0=RS_175._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col15"] - <-Map 20 [SIMPLE_EDGE] + <-Map 15 [SIMPLE_EDGE] SHUFFLE [RS_175] PartitionCols:_col0 Select Operator [SEL_167] (rows=36524 width=1119) @@ -493,12 +522,12 @@ Stage-0 Filter Operator [FIL_571] (rows=36524 width=1119) predicate:((d_year = 2001) and d_date_sk is not null) Please refer to the previous TableScan [TS_3] - <-Reducer 44 [SIMPLE_EDGE] + <-Reducer 35 [SIMPLE_EDGE] SHUFFLE [RS_174] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_604] (rows=633595212 width=88) Conds:RS_171._col1=RS_172._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col15"] - <-Map 42 [SIMPLE_EDGE] + <-Map 34 [SIMPLE_EDGE] SHUFFLE [RS_171] PartitionCols:_col1 Select Operator [SEL_33] (rows=575995635 width=88) @@ -507,7 +536,7 @@ Stage-0 predicate:(ss_addr_sk is not null and ss_cdemo_sk is not null and ss_customer_sk is not null and ss_hdemo_sk is not null and ss_item_sk is not null and ss_promo_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_ticket_number is not null) TableScan [TS_31] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_wholesale_cost","ss_list_price","ss_coupon_amt"] - <-Map 45 [SIMPLE_EDGE] + <-Map 36 [SIMPLE_EDGE] SHUFFLE [RS_172] PartitionCols:_col0 Select Operator [SEL_36] (rows=2851 width=1436) @@ -516,55 +545,6 @@ Stage-0 predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null) TableScan [TS_34] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_color","i_product_name"] - <-Reducer 39 [SIMPLE_EDGE] - SHUFFLE [RS_209] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_603] (rows=7920 width=107) - Conds:RS_206._col1=RS_207._col0(Inner),Output:["_col0"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_206] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_11] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_207] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_14] - <-Reducer 51 [ONE_TO_ONE_EDGE] - FORWARD [RS_213] - PartitionCols:_col0 - Select Operator [SEL_196] (rows=52798137 width=135) - Output:["_col0"] - Filter Operator [FIL_195] (rows=52798137 width=135) - predicate:(_col1 > (2 * _col2)) - Group By Operator [GBY_194] (rows=158394413 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 50 [SIMPLE_EDGE] - SHUFFLE [RS_193] - PartitionCols:_col0 - Group By Operator [GBY_192] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0 - Select Operator [SEL_190] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_607] (rows=316788826 width=135) - Conds:RS_187._col0, _col1=RS_188._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_187] - PartitionCols:_col0, _col1 - Select Operator [SEL_55] (rows=287989836 width=135) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_554] (rows=287989836 width=135) - predicate:(cs_item_sk is not null and cs_order_number is not null) - TableScan [TS_53] (rows=287989836 width=135) - default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"] - <-Map 52 [SIMPLE_EDGE] - SHUFFLE [RS_188] - PartitionCols:_col0, _col1 - Select Operator [SEL_58] (rows=28798881 width=106) - Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_555] (rows=28798881 width=106) - predicate:(cr_item_sk is not null and cr_order_number is not null) - TableScan [TS_56] (rows=28798881 width=106) - default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"] <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_256] PartitionCols:_col2, _col1, _col3 @@ -583,7 +563,7 @@ Stage-0 predicate:(_col56 <> _col19) Merge Join Operator [MERGEJOIN_596] (rows=1493984654 width=88) Conds:RS_118._col37=RS_119._col0(Inner),Output:["_col7","_col9","_col14","_col15","_col16","_col17","_col19","_col23","_col24","_col25","_col26","_col28","_col29","_col43","_col44","_col45","_col46","_col49","_col56"] - <-Map 55 [SIMPLE_EDGE] + <-Map 44 [SIMPLE_EDGE] SHUFFLE [RS_119] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_99] @@ -592,62 +572,74 @@ Stage-0 PartitionCols:_col37 Merge Join Operator [MERGEJOIN_595] (rows=1358167838 width=88) Conds:RS_115._col0=RS_116._col16(Inner),Output:["_col7","_col9","_col14","_col15","_col16","_col17","_col19","_col23","_col24","_col25","_col26","_col28","_col29","_col37","_col43","_col44","_col45","_col46","_col49"] - <-Reducer 27 [SIMPLE_EDGE] + <-Reducer 6 [SIMPLE_EDGE] + SHUFFLE [RS_115] + PartitionCols:_col0 + Please refer to the previous Merge Join Operator [MERGEJOIN_584] + <-Reducer 22 [SIMPLE_EDGE] SHUFFLE [RS_116] PartitionCols:_col16 Select Operator [SEL_96] (rows=1234698008 width=88) Output:["_col3","_col4","_col5","_col6","_col8","_col9","_col16","_col17","_col23","_col24","_col25","_col26","_col29"] Merge Join Operator [MERGEJOIN_594] (rows=1234698008 width=88) Conds:RS_93._col5, _col12=RS_94._col0, _col1(Inner),Output:["_col6","_col7","_col13","_col14","_col15","_col16","_col19","_col26","_col27","_col29","_col30","_col31","_col32"] - <-Map 54 [SIMPLE_EDGE] + <-Map 43 [SIMPLE_EDGE] SHUFFLE [RS_94] PartitionCols:_col0, _col1 Please refer to the previous Select Operator [SEL_77] - <-Reducer 26 [SIMPLE_EDGE] + <-Reducer 21 [SIMPLE_EDGE] SHUFFLE [RS_93] PartitionCols:_col5, _col12 Merge Join Operator [MERGEJOIN_593] (rows=1122452711 width=88) Conds:RS_90._col9=RS_91._col0(Inner),Output:["_col5","_col6","_col7","_col12","_col13","_col14","_col15","_col16","_col19","_col26","_col27","_col29","_col30","_col31","_col32"] - <-Map 41 [SIMPLE_EDGE] + <-Map 33 [SIMPLE_EDGE] SHUFFLE [RS_91] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_21] - <-Reducer 25 [SIMPLE_EDGE] + <-Reducer 20 [SIMPLE_EDGE] SHUFFLE [RS_90] PartitionCols:_col9 Merge Join Operator [MERGEJOIN_592] (rows=1020411534 width=88) Conds:RS_87._col10=RS_88._col0(Inner),Output:["_col5","_col6","_col7","_col9","_col12","_col13","_col14","_col15","_col16","_col19","_col26","_col27"] - <-Map 53 [SIMPLE_EDGE] + <-Map 42 [SIMPLE_EDGE] SHUFFLE [RS_88] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_71] - <-Reducer 24 [SIMPLE_EDGE] + <-Reducer 19 [SIMPLE_EDGE] SHUFFLE [RS_87] PartitionCols:_col10 Merge Join Operator [MERGEJOIN_591] (rows=927646829 width=88) Conds:RS_84._col5=RS_85._col0(Inner),Output:["_col5","_col6","_col7","_col9","_col10","_col12","_col13","_col14","_col15","_col16","_col19"] - <-Reducer 23 [SIMPLE_EDGE] + <-Reducer 40 [ONE_TO_ONE_EDGE] + FORWARD [RS_85] + PartitionCols:_col0 + Please refer to the previous Select Operator [SEL_196] + <-Reducer 18 [SIMPLE_EDGE] SHUFFLE [RS_84] PartitionCols:_col5 Merge Join Operator [MERGEJOIN_590] (rows=843315281 width=88) Conds:RS_81._col0=RS_82._col5(Inner),Output:["_col5","_col6","_col7","_col9","_col10","_col12","_col13","_col14","_col15","_col16","_col19"] - <-Reducer 22 [SIMPLE_EDGE] + <-Reducer 31 [SIMPLE_EDGE] + SHUFFLE [RS_81] + PartitionCols:_col0 + Please refer to the previous Merge Join Operator [MERGEJOIN_603] + <-Reducer 17 [SIMPLE_EDGE] SHUFFLE [RS_82] PartitionCols:_col5 Select Operator [SEL_52] (rows=766650239 width=88) Output:["_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col13","_col16"] Merge Join Operator [MERGEJOIN_588] (rows=766650239 width=88) Conds:RS_49._col7=RS_50._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col8","_col9","_col10","_col11","_col12","_col15"] - <-Map 46 [SIMPLE_EDGE] + <-Map 37 [SIMPLE_EDGE] SHUFFLE [RS_50] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_42] - <-Reducer 21 [SIMPLE_EDGE] + <-Reducer 16 [SIMPLE_EDGE] SHUFFLE [RS_49] PartitionCols:_col7 Merge Join Operator [MERGEJOIN_587] (rows=696954748 width=88) Conds:RS_46._col0=RS_47._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col15"] - <-Map 20 [SIMPLE_EDGE] + <-Map 15 [SIMPLE_EDGE] SHUFFLE [RS_47] PartitionCols:_col0 Select Operator [SEL_39] (rows=36524 width=1119) @@ -655,118 +647,8 @@ Stage-0 Filter Operator [FIL_552] (rows=36524 width=1119) predicate:((d_year = 2000) and d_date_sk is not null) Please refer to the previous TableScan [TS_3] - <-Reducer 43 [SIMPLE_EDGE] + <-Reducer 35 [SIMPLE_EDGE] SHUFFLE [RS_46] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_586] (rows=633595212 width=88) - Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col15"] - <-Map 42 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_33] - <-Map 45 [SIMPLE_EDGE] - SHUFFLE [RS_44] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_36] - <-Reducer 37 [SIMPLE_EDGE] - SHUFFLE [RS_81] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_585] (rows=7920 width=107) - Conds:RS_78._col1=RS_79._col0(Inner),Output:["_col0"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_78] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_11] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_79] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_14] - <-Reducer 49 [ONE_TO_ONE_EDGE] - FORWARD [RS_85] - PartitionCols:_col0 - Select Operator [SEL_68] (rows=52798137 width=135) - Output:["_col0"] - Filter Operator [FIL_67] (rows=52798137 width=135) - predicate:(_col1 > (2 * _col2)) - Group By Operator [GBY_66] (rows=158394413 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 - <-Reducer 48 [SIMPLE_EDGE] - SHUFFLE [RS_65] - PartitionCols:_col0 - Group By Operator [GBY_64] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0 - Select Operator [SEL_62] (rows=316788826 width=135) - Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_589] (rows=316788826 width=135) - Conds:RS_59._col0, _col1=RS_60._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"] - <-Map 47 [SIMPLE_EDGE] - SHUFFLE [RS_59] - PartitionCols:_col0, _col1 - Please refer to the previous Select Operator [SEL_55] - <-Map 52 [SIMPLE_EDGE] - SHUFFLE [RS_60] - PartitionCols:_col0, _col1 - Please refer to the previous Select Operator [SEL_58] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_115] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_584] (rows=128840811 width=860) - Conds:RS_112._col1=RS_113._col0(Inner),Output:["_col0","_col7","_col9","_col14","_col15","_col16","_col17","_col19"] - <-Map 55 [SIMPLE_EDGE] - SHUFFLE [RS_113] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_99] - <-Reducer 5 [SIMPLE_EDGE] - SHUFFLE [RS_112] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_583] (rows=117128008 width=860) - Conds:RS_109._col3=RS_110._col0(Inner),Output:["_col0","_col1","_col7","_col9","_col14","_col15","_col16","_col17"] - <-Map 41 [SIMPLE_EDGE] - SHUFFLE [RS_110] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_21] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_109] - PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_582] (rows=106480005 width=860) - Conds:RS_106._col2=RS_107._col0(Inner),Output:["_col0","_col1","_col3","_col7","_col9"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_106] - PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_580] (rows=96800003 width=860) - Conds:RS_103._col4=RS_104._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col7","_col9"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_104] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=73049 width=1119) - Output:["_col0","_col1"] - Filter Operator [FIL_542] (rows=73049 width=1119) - predicate:d_date_sk is not null - Please refer to the previous TableScan [TS_3] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_103] - PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_579] (rows=88000001 width=860) - Conds:RS_100._col5=RS_101._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7"] - <-Map 20 [SIMPLE_EDGE] - SHUFFLE [RS_101] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_5] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_100] - PartitionCols:_col5 - Please refer to the previous Select Operator [SEL_2] - <-Reducer 36 [SIMPLE_EDGE] - SHUFFLE [RS_107] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_581] (rows=7920 width=107) - Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col0"] - <-Map 35 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_11] - <-Map 40 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_14] + Please refer to the previous Merge Join Operator [MERGEJOIN_604] diff --git a/ql/src/test/results/clientpositive/perf/tez/query70.q.out b/ql/src/test/results/clientpositive/perf/tez/query70.q.out index 0f6891e75b..612fe6b8e3 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query70.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query70.q.out @@ -75,28 +75,27 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) +Reducer 10 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 11 (SIMPLE_EDGE) Reducer 3 <- Map 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +Reducer 6 <- Reducer 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) -Reducer 9 <- Map 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +Reducer 9 <- Reducer 8 (SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 + Reducer 9 File Output Operator [FS_61] Limit [LIM_59] (rows=100 width=88) Number of rows:100 Select Operator [SEL_58] (rows=1149975358 width=88) Output:["_col0","_col1","_col2","_col3","_col4"] - <-Reducer 6 [SIMPLE_EDGE] + <-Reducer 8 [SIMPLE_EDGE] SHUFFLE [RS_57] Select Operator [SEL_55] (rows=1149975358 width=88) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] @@ -104,14 +103,14 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col2 DESC NULLS LAST","partition by:":"(grouping(_col3, 1) + grouping(_col3, 0)), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END"}] Select Operator [SEL_53] (rows=1149975358 width=88) Output:["_col0","_col1","_col2","_col3"] - <-Reducer 5 [SIMPLE_EDGE] + <-Reducer 7 [SIMPLE_EDGE] SHUFFLE [RS_52] PartitionCols:(grouping(_col3, 1) + grouping(_col3, 0)), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END Select Operator [SEL_51] (rows=1149975358 width=88) Output:["_col0","_col1","_col2","_col3"] Group By Operator [GBY_50] (rows=1149975358 width=88) Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2 - <-Reducer 4 [SIMPLE_EDGE] + <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_49] PartitionCols:_col0, _col1, _col2 Group By Operator [GBY_48] (rows=2299950717 width=88) @@ -120,7 +119,44 @@ Stage-0 Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_89] (rows=766650239 width=88) Conds:RS_43._col7=RS_44._col0(Inner),Output:["_col2","_col6","_col7"] - <-Reducer 11 [SIMPLE_EDGE] + <-Reducer 10 [SIMPLE_EDGE] + SHUFFLE [RS_43] + PartitionCols:_col7 + Merge Join Operator [MERGEJOIN_86] (rows=696954748 width=88) + Conds:RS_40._col1=RS_41._col0(Inner),Output:["_col2","_col6","_col7"] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_40] + PartitionCols:_col1 + Merge Join Operator [MERGEJOIN_87] (rows=633595212 width=88) + Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_18] + PartitionCols:_col0 + Select Operator [SEL_2] (rows=575995635 width=88) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_78] (rows=575995635 width=88) + predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) + TableScan [TS_0] (rows=575995635 width=88) + default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] + <-Map 11 [SIMPLE_EDGE] + SHUFFLE [RS_19] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=8116 width=1119) + Output:["_col0"] + Filter Operator [FIL_79] (rows=8116 width=1119) + predicate:(d_date_sk is not null and d_month_seq BETWEEN 1212 AND 1223) + TableScan [TS_3] (rows=73049 width=1119) + default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_41] + PartitionCols:_col0 + Select Operator [SEL_8] (rows=1704 width=1910) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_80] (rows=1704 width=1910) + predicate:(s_state is not null and s_store_sk is not null) + TableScan [TS_6] (rows=1704 width=1910) + default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county","s_state"] + <-Reducer 5 [SIMPLE_EDGE] SHUFFLE [RS_44] PartitionCols:_col0 Select Operator [SEL_32] (rows=116159124 width=88) @@ -131,19 +167,23 @@ Stage-0 Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 DESC NULLS LAST","partition by:":"_col0"}] Select Operator [SEL_30] (rows=348477374 width=88) Output:["_col0","_col1"] - <-Reducer 10 [SIMPLE_EDGE] + <-Reducer 4 [SIMPLE_EDGE] SHUFFLE [RS_29] PartitionCols:_col0 Group By Operator [GBY_27] (rows=348477374 width=88) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 - <-Reducer 9 [SIMPLE_EDGE] + <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_26] PartitionCols:_col0 Group By Operator [GBY_25] (rows=696954748 width=88) Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col6 Merge Join Operator [MERGEJOIN_88] (rows=696954748 width=88) Conds:RS_21._col1=RS_22._col0(Inner),Output:["_col2","_col6"] - <-Map 14 [SIMPLE_EDGE] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_21] + PartitionCols:_col1 + Please refer to the previous Merge Join Operator [MERGEJOIN_87] + <-Map 13 [SIMPLE_EDGE] SHUFFLE [RS_22] PartitionCols:_col0 Select Operator [SEL_17] (rows=1704 width=1910) @@ -152,54 +192,4 @@ Stage-0 predicate:(s_state is not null and s_store_sk is not null) TableScan [TS_15] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_21] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_87] (rows=633595212 width=88) - Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Select Operator [SEL_2] (rows=575995635 width=88) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_78] (rows=575995635 width=88) - predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) - TableScan [TS_0] (rows=575995635 width=88) - default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_19] - PartitionCols:_col0 - Select Operator [SEL_5] (rows=8116 width=1119) - Output:["_col0"] - Filter Operator [FIL_79] (rows=8116 width=1119) - predicate:(d_date_sk is not null and d_month_seq BETWEEN 1212 AND 1223) - TableScan [TS_3] (rows=73049 width=1119) - default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_43] - PartitionCols:_col7 - Merge Join Operator [MERGEJOIN_86] (rows=696954748 width=88) - Conds:RS_40._col1=RS_41._col0(Inner),Output:["_col2","_col6","_col7"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_41] - PartitionCols:_col0 - Select Operator [SEL_8] (rows=1704 width=1910) - Output:["_col0","_col1","_col2"] - Filter Operator [FIL_80] (rows=1704 width=1910) - predicate:(s_state is not null and s_store_sk is not null) - TableScan [TS_6] (rows=1704 width=1910) - default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county","s_state"] - <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_40] - PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_85] (rows=633595212 width=88) - Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_37] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_2] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_38] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_5] diff --git a/ql/src/test/results/clientpositive/perf/tez/query83.q.out b/ql/src/test/results/clientpositive/perf/tez/query83.q.out index 2a3a946e2e..67949894aa 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query83.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query83.q.out @@ -134,25 +134,19 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 10 <- Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Map 27 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) +Reducer 11 <- Map 21 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) +Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 20 (ONE_TO_ONE_EDGE) -Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 22 (ONE_TO_ONE_EDGE) -Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 24 (ONE_TO_ONE_EDGE) -Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 25 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 18 (ONE_TO_ONE_EDGE) +Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) +Reducer 18 <- Reducer 17 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 20 <- Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Map 18 (SIMPLE_EDGE), Map 25 (SIMPLE_EDGE) -Reducer 22 <- Reducer 21 (SIMPLE_EDGE) -Reducer 23 <- Map 18 (SIMPLE_EDGE), Map 25 (SIMPLE_EDGE) -Reducer 24 <- Reducer 23 (SIMPLE_EDGE) Reducer 3 <- Reducer 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 8 <- Map 26 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) -Reducer 9 <- Reducer 16 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 8 <- Map 20 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) +Reducer 9 <- Reducer 15 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Stage-0 Fetch Operator @@ -182,13 +176,13 @@ Stage-0 Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4 Merge Join Operator [MERGEJOIN_220] (rows=69685294 width=77) Conds:RS_72._col0=RS_73._col0(Inner),Output:["_col2","_col4"] - <-Reducer 16 [SIMPLE_EDGE] + <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_73] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_215] (rows=80353 width=1119) - Conds:RS_65._col1=RS_66._col0(Inner),Output:["_col0"] + Merge Join Operator [MERGEJOIN_218] (rows=80353 width=1119) + Conds:RS_105._col1=RS_106._col0(Inner),Output:["_col0"] <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_65] + SHUFFLE [RS_105] PartitionCols:_col1 Select Operator [SEL_8] (rows=73049 width=1119) Output:["_col0","_col1"] @@ -196,19 +190,19 @@ Stage-0 predicate:(d_date is not null and d_date_sk is not null) TableScan [TS_6] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 22 [ONE_TO_ONE_EDGE] - FORWARD [RS_66] + <-Reducer 18 [ONE_TO_ONE_EDGE] + FORWARD [RS_106] PartitionCols:_col0 - Group By Operator [GBY_63] (rows=40176 width=1119) + Group By Operator [GBY_103] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 - <-Reducer 21 [SIMPLE_EDGE] - SHUFFLE [RS_62] + <-Reducer 17 [SIMPLE_EDGE] + SHUFFLE [RS_102] PartitionCols:_col0 Group By Operator [GBY_61] (rows=80353 width=1119) Output:["_col0"],keys:_col0 Merge Join Operator [MERGEJOIN_214] (rows=80353 width=1119) Conds:RS_57._col1=RS_58._col0(Left Semi),Output:["_col0"] - <-Map 18 [SIMPLE_EDGE] + <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_57] PartitionCols:_col1 Select Operator [SEL_11] (rows=73049 width=1119) @@ -217,7 +211,7 @@ Stage-0 predicate:(d_date is not null and d_week_seq is not null) TableScan [TS_9] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"] - <-Map 25 [SIMPLE_EDGE] + <-Map 19 [SIMPLE_EDGE] SHUFFLE [RS_58] PartitionCols:_col0 Group By Operator [GBY_16] (rows=36525 width=1119) @@ -242,7 +236,7 @@ Stage-0 predicate:(i_item_id is not null and i_item_sk is not null) TableScan [TS_3] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Map 26 [SIMPLE_EDGE] + <-Map 20 [SIMPLE_EDGE] SHUFFLE [RS_69] PartitionCols:_col1 Select Operator [SEL_42] (rows=57591150 width=77) @@ -263,6 +257,10 @@ Stage-0 Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4 Merge Join Operator [MERGEJOIN_221] (rows=17422145 width=92) Conds:RS_112._col0=RS_113._col0(Inner),Output:["_col2","_col4"] + <-Reducer 15 [SIMPLE_EDGE] + SHUFFLE [RS_113] + PartitionCols:_col0 + Please refer to the previous Merge Join Operator [MERGEJOIN_218] <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_112] PartitionCols:_col0 @@ -272,7 +270,7 @@ Stage-0 SHUFFLE [RS_110] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_5] - <-Map 27 [SIMPLE_EDGE] + <-Map 21 [SIMPLE_EDGE] SHUFFLE [RS_109] PartitionCols:_col1 Select Operator [SEL_82] (rows=14398467 width=92) @@ -281,35 +279,6 @@ Stage-0 predicate:(wr_item_sk is not null and wr_returned_date_sk is not null) TableScan [TS_80] (rows=14398467 width=92) default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_item_sk","wr_return_quantity"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_113] - PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_218] (rows=80353 width=1119) - Conds:RS_105._col1=RS_106._col0(Inner),Output:["_col0"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_105] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_8] - <-Reducer 24 [ONE_TO_ONE_EDGE] - FORWARD [RS_106] - PartitionCols:_col0 - Group By Operator [GBY_103] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 23 [SIMPLE_EDGE] - SHUFFLE [RS_102] - PartitionCols:_col0 - Group By Operator [GBY_101] (rows=80353 width=1119) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_217] (rows=80353 width=1119) - Conds:RS_97._col1=RS_98._col0(Left Semi),Output:["_col0"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_97] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_11] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_98] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_16] <-Reducer 4 [ONE_TO_ONE_EDGE] FORWARD [RS_120] PartitionCols:_col0 @@ -325,32 +294,7 @@ Stage-0 <-Reducer 15 [SIMPLE_EDGE] SHUFFLE [RS_33] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_212] (rows=80353 width=1119) - Conds:RS_25._col1=RS_26._col0(Inner),Output:["_col0"] - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_25] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_8] - <-Reducer 20 [ONE_TO_ONE_EDGE] - FORWARD [RS_26] - PartitionCols:_col0 - Group By Operator [GBY_23] (rows=40176 width=1119) - Output:["_col0"],keys:KEY._col0 - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_22] - PartitionCols:_col0 - Group By Operator [GBY_21] (rows=80353 width=1119) - Output:["_col0"],keys:_col0 - Merge Join Operator [MERGEJOIN_211] (rows=80353 width=1119) - Conds:RS_17._col1=RS_18._col0(Left Semi),Output:["_col0"] - <-Map 18 [SIMPLE_EDGE] - SHUFFLE [RS_17] - PartitionCols:_col1 - Please refer to the previous Select Operator [SEL_11] - <-Map 25 [SIMPLE_EDGE] - SHUFFLE [RS_18] - PartitionCols:_col0 - Please refer to the previous Group By Operator [GBY_16] + Please refer to the previous Merge Join Operator [MERGEJOIN_218] <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_32] PartitionCols:_col0 diff --git a/ql/src/test/results/clientpositive/perf/tez/query90.q.out b/ql/src/test/results/clientpositive/perf/tez/query90.q.out index d63ce1b28b..fbd7692645 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query90.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query90.q.out @@ -44,15 +44,14 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) -Reducer 11 <- Reducer 10 (CUSTOM_SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) -Reducer 3 <- Map 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Map 14 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 10 <- Reducer 9 (CUSTOM_SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 11 (SIMPLE_EDGE) +Reducer 3 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 4 <- Map 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Reducer 6 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) +Reducer 6 <- Reducer 10 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) +Reducer 8 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 9 <- Map 13 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Stage-0 @@ -71,18 +70,18 @@ Stage-0 Output:["_col0"] Merge Join Operator [MERGEJOIN_92] (rows=1 width=17) Conds:(Inner),Output:["_col0","_col1"] - <-Reducer 11 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_53] - Group By Operator [GBY_50] (rows=1 width=8) + <-Reducer 10 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_52] + Group By Operator [GBY_24] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] - <-Reducer 10 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_49] - Group By Operator [GBY_48] (rows=1 width=8) + <-Reducer 9 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_23] + Group By Operator [GBY_22] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_91] (rows=191667562 width=135) - Conds:RS_44._col1=RS_45._col0(Inner) - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_45] + Merge Join Operator [MERGEJOIN_88] (rows=191667562 width=135) + Conds:RS_18._col1=RS_19._col0(Inner) + <-Map 13 [SIMPLE_EDGE] + SHUFFLE [RS_19] PartitionCols:_col0 Select Operator [SEL_11] (rows=3600 width=107) Output:["_col0"] @@ -90,22 +89,22 @@ Stage-0 predicate:((hd_dep_count = 8) and hd_demo_sk is not null) TableScan [TS_9] (rows=7200 width=107) default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_44] + <-Reducer 8 [SIMPLE_EDGE] + SHUFFLE [RS_18] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_90] (rows=174243235 width=135) - Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_42] + Merge Join Operator [MERGEJOIN_87] (rows=174243235 width=135) + Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1"] + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_16] PartitionCols:_col0 - Select Operator [SEL_34] (rows=9600 width=471) + Select Operator [SEL_8] (rows=9600 width=471) Output:["_col0"] - Filter Operator [FIL_84] (rows=9600 width=471) - predicate:(t_hour BETWEEN 14 AND 15 and t_time_sk is not null) + Filter Operator [FIL_80] (rows=9600 width=471) + predicate:(t_hour BETWEEN 6 AND 7 and t_time_sk is not null) TableScan [TS_6] (rows=86400 width=471) default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_41] + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_15] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_89] (rows=158402938 width=135) Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col0","_col1"] @@ -118,7 +117,7 @@ Stage-0 predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null) TableScan [TS_0] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"] - <-Map 12 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] SHUFFLE [RS_39] PartitionCols:_col0 Select Operator [SEL_5] (rows=511 width=585) @@ -128,43 +127,34 @@ Stage-0 TableScan [TS_3] (rows=4602 width=585) default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk","wp_char_count"] <-Reducer 5 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_52] - Group By Operator [GBY_24] (rows=1 width=8) + PARTITION_ONLY_SHUFFLE [RS_53] + Group By Operator [GBY_50] (rows=1 width=8) Output:["_col0"],aggregations:["count(VALUE._col0)"] <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_23] - Group By Operator [GBY_22] (rows=1 width=8) + PARTITION_ONLY_SHUFFLE [RS_49] + Group By Operator [GBY_48] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Merge Join Operator [MERGEJOIN_88] (rows=191667562 width=135) - Conds:RS_18._col1=RS_19._col0(Inner) - <-Map 14 [SIMPLE_EDGE] - SHUFFLE [RS_19] + Merge Join Operator [MERGEJOIN_91] (rows=191667562 width=135) + Conds:RS_44._col1=RS_45._col0(Inner) + <-Map 13 [SIMPLE_EDGE] + SHUFFLE [RS_45] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_11] <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_18] + SHUFFLE [RS_44] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_87] (rows=174243235 width=135) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1"] - <-Map 13 [SIMPLE_EDGE] - SHUFFLE [RS_16] + Merge Join Operator [MERGEJOIN_90] (rows=174243235 width=135) + Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1"] + <-Map 12 [SIMPLE_EDGE] + SHUFFLE [RS_42] PartitionCols:_col0 - Select Operator [SEL_8] (rows=9600 width=471) + Select Operator [SEL_34] (rows=9600 width=471) Output:["_col0"] - Filter Operator [FIL_80] (rows=9600 width=471) - predicate:(t_hour BETWEEN 6 AND 7 and t_time_sk is not null) + Filter Operator [FIL_84] (rows=9600 width=471) + predicate:(t_hour BETWEEN 14 AND 15 and t_time_sk is not null) Please refer to the previous TableScan [TS_6] <-Reducer 2 [SIMPLE_EDGE] - SHUFFLE [RS_15] + SHUFFLE [RS_41] PartitionCols:_col0 - Merge Join Operator [MERGEJOIN_86] (rows=158402938 width=135) - Conds:RS_12._col2=RS_13._col0(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col2 - Please refer to the previous Select Operator [SEL_2] - <-Map 12 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_5] + Please refer to the previous Merge Join Operator [MERGEJOIN_89] diff --git a/ql/src/test/results/clientpositive/perf/tez/query92.q.out b/ql/src/test/results/clientpositive/perf/tez/query92.q.out index c97c3cf040..fbb4eb3a55 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query92.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query92.q.out @@ -59,13 +59,12 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -Reducer 6 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) -Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 10 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) +Reducer 6 <- Reducer 2 (SIMPLE_EDGE) +Reducer 7 <- Map 9 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Stage-0 Fetch Operator @@ -107,7 +106,7 @@ Stage-0 predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) TableScan [TS_0] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_ext_discount_amt"] - <-Map 9 [SIMPLE_EDGE] + <-Map 8 [SIMPLE_EDGE] SHUFFLE [RS_28] PartitionCols:_col0 Select Operator [SEL_5] (rows=8116 width=1119) @@ -116,12 +115,12 @@ Stage-0 predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-03-18 00:00:00.0 AND 1998-06-16 01:00:00.0 and d_date_sk is not null) TableScan [TS_3] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 8 [ONE_TO_ONE_EDGE] + <-Reducer 7 [ONE_TO_ONE_EDGE] FORWARD [RS_31] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_63] (rows=87121617 width=135) Conds:RS_23._col1=RS_24._col0(Inner),Output:["_col0","_col2"] - <-Map 10 [SIMPLE_EDGE] + <-Map 9 [SIMPLE_EDGE] SHUFFLE [RS_24] PartitionCols:_col0 Select Operator [SEL_22] (rows=231000 width=1436) @@ -130,26 +129,17 @@ Stage-0 predicate:((i_manufact_id = 269) and i_item_sk is not null) TableScan [TS_20] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 7 [ONE_TO_ONE_EDGE] + <-Reducer 6 [ONE_TO_ONE_EDGE] FORWARD [RS_23] PartitionCols:_col1 Select Operator [SEL_19] (rows=79201469 width=135) Output:["_col0","_col1"] Group By Operator [GBY_18] (rows=79201469 width=135) Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0 - <-Reducer 6 [SIMPLE_EDGE] + <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_17] PartitionCols:_col0 Group By Operator [GBY_16] (rows=158402938 width=135) Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col1 - Merge Join Operator [MERGEJOIN_62] (rows=158402938 width=135) - Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_12] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_2] - <-Map 9 [SIMPLE_EDGE] - SHUFFLE [RS_13] - PartitionCols:_col0 - Please refer to the previous Select Operator [SEL_5] + Please refer to the previous Merge Join Operator [MERGEJOIN_61]