diff --git common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java index 799355a..b7dc88c 100644 --- common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java +++ common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java @@ -74,7 +74,7 @@ public VertexType vertexType; public static enum EdgeType { - BROADCAST, SHUFFLE, MULTICAST, PARTITION_ONLY_SHUFFLE, UNKNOWN + BROADCAST, SHUFFLE, MULTICAST, PARTITION_ONLY_SHUFFLE, FORWARD, UNKNOWN }; public String edgeType; diff --git common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java index 294dc6b..69e5358 100644 --- common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java +++ common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java @@ -34,6 +34,8 @@ public String mapEdgeType(String edgeName) { return "PARTITION_ONLY_SHUFFLE"; case "CUSTOM_EDGE": return "MULTICAST"; + case "ONE_TO_ONE_EDGE": + return "FORWARD"; default: return "UNKNOWN"; } diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 9c954be..c8d8b46 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1601,6 +1601,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Reduce deduplication merges two RSs by moving key/parts/reducer-num of the child RS to parent RS. \n" + "That means if reducer-num of the child RS is fixed (order by or forced bucketing) and small, it can make very slow, single MR.\n" + "The optimization will be automatically disabled if number of reducers would be less than specified value."), + HIVEOPTJOINREDUCEDEDUPLICATION("hive.optimize.joinreducededuplication", true, + "Remove extra shuffle/sorting operations after join algorithm selection has been executed. \n" + + "Currently it only works with Apache Tez. This should always be set to true. \n" + + "Since it is a new feature, it has been made configurable."), HIVEOPTSORTDYNAMICPARTITION("hive.optimize.sort.dynamic.partition", false, "When enabled dynamic partitioning column will be globally sorted.\n" + diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 1cc0104..362a796 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -742,7 +742,8 @@ minillaplocal.query.files=acid_globallimit.q,\ groupby_resolution.q,\ windowing_windowspec2.q,\ vectorized_join46.q,\ - vectorized_multi_output_select.q + vectorized_multi_output_select.q,\ + partialdhj.q encrypted.query.files=encryption_join_unencrypted_tbl.q,\ encryption_insert_partition_static.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 51beaad..64e0d9f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -293,6 +293,10 @@ public GroupInputEdge createEdge(VertexGroup group, JobConf vConf, Vertex w, mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; + case ONE_TO_ONE_EDGE: + mergeInputClass = ConcatenatedMergedKeyValueInput.class; + break; + case SIMPLE_EDGE: setupAutoReducerParallelism(edgeProp, w); // fall through @@ -398,18 +402,26 @@ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); return et3Conf.createDefaultEdgeProperty(); + case ONE_TO_ONE_EDGE: + UnorderedKVEdgeConfig et4Conf = UnorderedKVEdgeConfig + .newBuilder(keyClass, valClass) + .setFromConfiguration(conf) + .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null) + .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) + .build(); + return et4Conf.createDefaultOneToOneEdgeProperty(); case SIMPLE_EDGE: default: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); - OrderedPartitionedKVEdgeConfig et4Conf = OrderedPartitionedKVEdgeConfig + OrderedPartitionedKVEdgeConfig et5Conf = OrderedPartitionedKVEdgeConfig .newBuilder(keyClass, valClass, MRPartitioner.class.getName(), partitionerConf) .setFromConfiguration(conf) .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), TezBytesComparator.class.getName(), null) .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); - return et4Conf.createDefaultEdgeProperty(); + return et5Conf.createDefaultEdgeProperty(); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java index d84a1e6..0538176 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java @@ -1073,8 +1073,11 @@ public static MapJoinDesc getMapJoinDesc(HiveConf hconf, ExprNodeDesc expr = colExprMap.get(column.getInternalName()); int index = ExprNodeDescUtils.indexOf(expr, values); if (index >= 0) { - colExprMap.put(column.getInternalName(), newValues.get(index)); schema.set(i, null); + if (adjustParentsChildren) { + // Since we remove reduce sink parents, replace original expressions + colExprMap.put(column.getInternalName(), newValues.get(index)); + } } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java index ac234d0..70919a6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java @@ -270,7 +270,11 @@ public static Object processReduceSinkToHashJoin(ReduceSinkOperator parentRS, Ma } } } else if (mapJoinOp.getConf().isDynamicPartitionHashJoin()) { - edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + if (parentRS.getConf().isForwarding()) { + edgeType = EdgeType.ONE_TO_ONE_EDGE; + } else { + edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + } } if (edgeType == EdgeType.CUSTOM_EDGE) { // disable auto parallelism for bucket map joins diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java index 388399c..c63c28d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java @@ -29,8 +29,8 @@ import java.util.Map.Entry; import java.util.Set; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.ForwardOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; @@ -43,16 +43,15 @@ import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.Utilities.ReduceField; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication.ReduceSinkDeduplicateProcCtx; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AggregationDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.GroupByDesc; +import org.apache.hadoop.hive.ql.plan.GroupByDesc.Mode; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -271,6 +270,52 @@ protected static int indexOf(ExprNodeDesc cexpr, ExprNodeDesc[] pexprs, Operator return result; } + protected static > T findFirstPossibleParent( + Operator start, Class target, boolean trustScript) throws SemanticException { + // Preserve only partitioning + return findFirstPossibleParent(start, target, trustScript, false); + } + + protected static > T findFirstPossibleParentPreserveSortOrder( + Operator start, Class target, boolean trustScript) throws SemanticException { + // Preserve partitioning and ordering + return findFirstPossibleParent(start, target, trustScript, true); + } + + @SuppressWarnings("unchecked") + private static > T findFirstPossibleParent( + Operator start, Class target, boolean trustScript, boolean preserveSortOrder) + throws SemanticException { + Operator cursor = CorrelationUtilities.getSingleParent(start); + for (; cursor != null; cursor = CorrelationUtilities.getSingleParent(cursor)) { + if (target.isAssignableFrom(cursor.getClass())) { + return (T) cursor; + } + if (cursor instanceof CommonJoinOperator) { + for (Operator op : ((CommonJoinOperator) cursor).getParentOperators()) { + if (target.isAssignableFrom(op.getClass())) { + return (T) op; + } + } + return null; + } + if (cursor instanceof ScriptOperator && !trustScript) { + return null; + } + if (!(cursor instanceof SelectOperator + || cursor instanceof FilterOperator + || cursor instanceof ForwardOperator + || cursor instanceof ScriptOperator + || (cursor instanceof GroupByOperator + && (!preserveSortOrder + || ((GroupByOperator) cursor).getConf().getMode() != Mode.HASH)) // Not order preserving + || cursor instanceof ReduceSinkOperator)) { + return null; + } + } + return null; + } + /** * Search the query plan tree from startPoint to the bottom. If there is no ReduceSinkOperator * between startPoint and the corresponding TableScanOperator, return the corresponding diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java index 701bde4..154c607 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java @@ -23,18 +23,14 @@ import java.util.ArrayList; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Stack; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.PTFOperator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; @@ -49,13 +45,6 @@ import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.PlanUtils; -import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.TableDesc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,457 +173,6 @@ protected abstract Object process(ReduceSinkOperator cRS, ReduceSinkDeduplicateP protected abstract Object process(ReduceSinkOperator cRS, GroupByOperator cGBY, ReduceSinkDeduplicateProcCtx dedupCtx) throws SemanticException; - - // for JOIN-RS case, it's not possible generally to merge if child has - // less key/partition columns than parents - protected boolean merge(ReduceSinkOperator cRS, JoinOperator pJoin, int minReducer) - throws SemanticException { - List> parents = pJoin.getParentOperators(); - ReduceSinkOperator[] pRSs = parents.toArray(new ReduceSinkOperator[parents.size()]); - ReduceSinkDesc cRSc = cRS.getConf(); - for (ReduceSinkOperator pRSNs : pRSs) { - ReduceSinkDesc pRSNc = pRSNs.getConf(); - if (cRSc.getKeyCols().size() != pRSNc.getKeyCols().size()) { - return false; - } - if (cRSc.getPartitionCols().size() != pRSNc.getPartitionCols().size()) { - return false; - } - Integer moveReducerNumTo = checkNumReducer(cRSc.getNumReducers(), pRSNc.getNumReducers()); - if (moveReducerNumTo == null || - moveReducerNumTo > 0 && cRSc.getNumReducers() < minReducer) { - return false; - } - - Integer moveRSOrderTo = checkOrder(true, cRSc.getOrder(), pRSNc.getOrder(), - cRSc.getNullOrder(), pRSNc.getNullOrder()); - if (moveRSOrderTo == null) { - return false; - } - } - - boolean[] sorted = CorrelationUtilities.getSortedTags(pJoin); - - int cKeySize = cRSc.getKeyCols().size(); - for (int i = 0; i < cKeySize; i++) { - ExprNodeDesc cexpr = cRSc.getKeyCols().get(i); - ExprNodeDesc[] pexprs = new ExprNodeDesc[pRSs.length]; - for (int tag = 0; tag < pRSs.length; tag++) { - pexprs[tag] = pRSs[tag].getConf().getKeyCols().get(i); - } - int found = CorrelationUtilities.indexOf(cexpr, pexprs, cRS, pRSs, sorted); - if (found != i) { - return false; - } - } - int cPartSize = cRSc.getPartitionCols().size(); - for (int i = 0; i < cPartSize; i++) { - ExprNodeDesc cexpr = cRSc.getPartitionCols().get(i); - ExprNodeDesc[] pexprs = new ExprNodeDesc[pRSs.length]; - for (int tag = 0; tag < pRSs.length; tag++) { - pexprs[tag] = pRSs[tag].getConf().getPartitionCols().get(i); - } - int found = CorrelationUtilities.indexOf(cexpr, pexprs, cRS, pRSs, sorted); - if (found != i) { - return false; - } - } - - for (ReduceSinkOperator pRS : pRSs) { - pRS.getConf().setNumReducers(cRS.getConf().getNumReducers()); - } - - return true; - } - - /** - * Current RSDedup remove/replace child RS. For key columns, - * sorting order, and the number of reducers, copy - * more specific part of configurations of child RS to that of parent RS. - * For partitioning columns, if both child RS and parent RS have been assigned - * partitioning columns, we will choose the more general partitioning columns. - * If parent RS has not been assigned any partitioning column, we will use - * partitioning columns (if exist) of child RS. - */ - protected boolean merge(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer) - throws SemanticException { - int[] result = extractMergeDirections(cRS, pRS, minReducer); - if (result == null) { - return false; - } - - if (result[0] > 0) { - // The sorting columns of the child RS are more specific than - // those of the parent RS. Assign sorting columns of the child RS - // to the parent RS. - List childKCs = cRS.getConf().getKeyCols(); - pRS.getConf().setKeyCols(ExprNodeDescUtils.backtrack(childKCs, cRS, pRS)); - } - - if (result[1] < 0) { - // The partitioning columns of the parent RS are more specific than - // those of the child RS. - List childPCs = cRS.getConf().getPartitionCols(); - if (childPCs != null && !childPCs.isEmpty()) { - // If partitioning columns of the child RS are assigned, - // assign these to the partitioning columns of the parent RS. - pRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(childPCs, cRS, pRS)); - } - } else if (result[1] > 0) { - // The partitioning columns of the child RS are more specific than - // those of the parent RS. - List parentPCs = pRS.getConf().getPartitionCols(); - if (parentPCs == null || parentPCs.isEmpty()) { - // If partitioning columns of the parent RS are not assigned, - // assign partitioning columns of the child RS to the parent RS. - ArrayList childPCs = cRS.getConf().getPartitionCols(); - pRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(childPCs, cRS, pRS)); - } - } - - if (result[2] > 0) { - // The sorting order of the child RS is more specific than - // that of the parent RS. Assign the sorting order of the child RS - // to the parent RS. - if (result[0] <= 0) { - // Sorting columns of the parent RS are more specific than those of the - // child RS but Sorting order of the child RS is more specific than - // that of the parent RS. - throw new SemanticException("Sorting columns and order don't match. " + - "Try set " + HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION + "=false;"); - } - pRS.getConf().setOrder(cRS.getConf().getOrder()); - pRS.getConf().setNullOrder(cRS.getConf().getNullOrder()); - } else { - // The sorting order of the parent RS is more specific or they are equal. - // We will copy the order from the child RS, and then fill in the order - // of the rest of columns with the one taken from parent RS. - StringBuilder order = new StringBuilder(cRS.getConf().getOrder()); - StringBuilder orderNull = new StringBuilder(cRS.getConf().getNullOrder()); - order.append(pRS.getConf().getOrder().substring(order.length())); - orderNull.append(pRS.getConf().getNullOrder().substring(orderNull.length())); - pRS.getConf().setOrder(order.toString()); - pRS.getConf().setNullOrder(orderNull.toString()); - } - - if (result[3] > 0) { - // The number of reducers of the child RS is more specific than - // that of the parent RS. Assign the number of reducers of the child RS - // to the parent RS. - pRS.getConf().setNumReducers(cRS.getConf().getNumReducers()); - } - - if (result[4] > 0) { - // This case happens only when pRS key is empty in which case we can use - // number of distribution keys and key serialization info from cRS - if (pRS.getConf().getKeyCols() != null && pRS.getConf().getKeyCols().size() == 0 - && cRS.getConf().getKeyCols() != null && cRS.getConf().getKeyCols().size() == 0) { - // As setNumDistributionKeys is a subset of keycols, the size should - // be 0 too. This condition maybe too strict. We may extend it in the - // future. - TableDesc keyTable = PlanUtils.getReduceKeyTableDesc(new ArrayList(), pRS - .getConf().getOrder(), pRS.getConf().getNullOrder()); - pRS.getConf().setKeySerializeInfo(keyTable); - } - } - return true; - } - - /** - * Returns merge directions between two RSs for criterias (ordering, number of reducers, - * reducer keys, partition keys). Returns null if any of categories is not mergeable. - * - * Values for each index can be -1, 0, 1 - * 1. 0 means two configuration in the category is the same - * 2. for -1, configuration of parent RS is more specific than child RS - * 3. for 1, configuration of child RS is more specific than parent RS - */ - private int[] extractMergeDirections(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer) - throws SemanticException { - ReduceSinkDesc cConf = cRS.getConf(); - ReduceSinkDesc pConf = pRS.getConf(); - // If there is a PTF between cRS and pRS we cannot ignore the order direction - final boolean checkStrictEquality = isStrictEqualityNeeded(cRS, pRS); - Integer moveRSOrderTo = checkOrder(checkStrictEquality, cConf.getOrder(), pConf.getOrder(), - cConf.getNullOrder(), pConf.getNullOrder()); - if (moveRSOrderTo == null) { - return null; - } - // if cRS is being used for distinct - the two reduce sinks are incompatible - if (cConf.getDistinctColumnIndices().size() >= 2) { - return null; - } - Integer moveReducerNumTo = checkNumReducer(cConf.getNumReducers(), pConf.getNumReducers()); - if (moveReducerNumTo == null || - moveReducerNumTo > 0 && cConf.getNumReducers() < minReducer) { - return null; - } - List ckeys = cConf.getKeyCols(); - List pkeys = pConf.getKeyCols(); - Integer moveKeyColTo = checkExprs(ckeys, pkeys, cRS, pRS); - if (moveKeyColTo == null) { - return null; - } - List cpars = cConf.getPartitionCols(); - List ppars = pConf.getPartitionCols(); - Integer movePartitionColTo = checkExprs(cpars, ppars, cRS, pRS); - if (movePartitionColTo == null) { - return null; - } - Integer moveNumDistKeyTo = checkNumDistributionKey(cConf.getNumDistributionKeys(), - pConf.getNumDistributionKeys()); - return new int[] {moveKeyColTo, movePartitionColTo, moveRSOrderTo, - moveReducerNumTo, moveNumDistKeyTo}; - } - - private boolean isStrictEqualityNeeded(ReduceSinkOperator cRS, ReduceSinkOperator pRS) { - Operator parent = cRS.getParentOperators().get(0); - while (parent != pRS) { - assert parent.getNumParent() == 1; - if (parent instanceof PTFOperator) { - return true; - } - parent = parent.getParentOperators().get(0); - } - return false; - } - - private Integer checkNumDistributionKey(int cnd, int pnd) { - // number of distribution keys of cRS is chosen only when numDistKeys of pRS - // is 0 or less. In all other cases, distribution of the keys is based on - // the pRS which is more generic than cRS. - // Examples: - // case 1: if pRS sort key is (a, b) and cRS sort key is (a, b, c) and number of - // distribution keys are 2 and 3 resp. then after merge the sort keys will - // be (a, b, c) while the number of distribution keys will be 2. - // case 2: if pRS sort key is empty and number of distribution keys is 0 - // and if cRS sort key is (a, b) and number of distribution keys is 2 then - // after merge new sort key will be (a, b) and number of distribution keys - // will be 2. - if (pnd <= 0) { - return 1; - } - return 0; - } - - /** - * Overlapping part of keys should be the same between parent and child. - * And if child has more keys than parent, non-overlapping part of keys - * should be backtrackable to parent. - */ - private Integer checkExprs(List ckeys, List pkeys, - ReduceSinkOperator cRS, ReduceSinkOperator pRS) throws SemanticException { - // If ckeys or pkeys have constant node expressions avoid the merge. - for (ExprNodeDesc ck : ckeys) { - if (ck instanceof ExprNodeConstantDesc) { - return null; - } - } - for (ExprNodeDesc pk : pkeys) { - if (pk instanceof ExprNodeConstantDesc) { - return null; - } - } - - Integer moveKeyColTo = 0; - if (ckeys == null || ckeys.isEmpty()) { - if (pkeys != null && !pkeys.isEmpty()) { - moveKeyColTo = -1; - } - } else { - if (pkeys == null || pkeys.isEmpty()) { - for (ExprNodeDesc ckey : ckeys) { - if (ExprNodeDescUtils.backtrack(ckey, cRS, pRS) == null) { - // cKey is not present in parent - return null; - } - } - moveKeyColTo = 1; - } else { - moveKeyColTo = sameKeys(ckeys, pkeys, cRS, pRS); - } - } - return moveKeyColTo; - } - - // backtrack key exprs of child to parent and compare it with parent's - protected Integer sameKeys(List cexprs, List pexprs, - Operator child, Operator parent) throws SemanticException { - int common = Math.min(cexprs.size(), pexprs.size()); - int limit = Math.max(cexprs.size(), pexprs.size()); - int i = 0; - for (; i < common; i++) { - ExprNodeDesc pexpr = pexprs.get(i); - ExprNodeDesc cexpr = ExprNodeDescUtils.backtrack(cexprs.get(i), child, parent); - if (cexpr == null || !pexpr.isSame(cexpr)) { - return null; - } - } - for (; i < limit; i++) { - if (cexprs.size() > pexprs.size()) { - if (ExprNodeDescUtils.backtrack(cexprs.get(i), child, parent) == null) { - // cKey is not present in parent - return null; - } - } - } - return Integer.valueOf(cexprs.size()).compareTo(pexprs.size()); - } - - protected Integer checkOrder(boolean checkStrictEquality, String corder, String porder, - String cNullOrder, String pNullOrder) { - assert corder.length() == cNullOrder.length(); - assert porder.length() == pNullOrder.length(); - if (corder == null || corder.trim().equals("")) { - if (porder == null || porder.trim().equals("")) { - return 0; - } - return -1; - } - if (porder == null || porder.trim().equals("")) { - return 1; - } - corder = corder.trim(); - porder = porder.trim(); - if (checkStrictEquality) { - // order of overlapping keys should be exactly the same - cNullOrder = cNullOrder.trim(); - pNullOrder = pNullOrder.trim(); - int target = Math.min(corder.length(), porder.length()); - if (!corder.substring(0, target).equals(porder.substring(0, target)) || - !cNullOrder.substring(0, target).equals(pNullOrder.substring(0, target))) { - return null; - } - } - return Integer.valueOf(corder.length()).compareTo(porder.length()); - } - - /** - * If number of reducers for RS is -1, the RS can have any number of reducers. - * It's generally true except for order-by or forced bucketing cases. - * if both of num-reducers are not -1, those number should be the same. - */ - protected Integer checkNumReducer(int creduce, int preduce) { - if (creduce < 0) { - if (preduce < 0) { - return 0; - } - return -1; - } - if (preduce < 0) { - return 1; - } - if (creduce != preduce) { - return null; - } - return 0; - } - - protected boolean aggressiveDedup(ReduceSinkOperator cRS, ReduceSinkOperator pRS, - ReduceSinkDeduplicateProcCtx dedupCtx) throws SemanticException { - assert cRS.getNumParent() == 1; - - ReduceSinkDesc cConf = cRS.getConf(); - ReduceSinkDesc pConf = pRS.getConf(); - List cKeys = cConf.getKeyCols(); - List pKeys = pConf.getKeyCols(); - - // Check that in the path between cRS and pRS, there are only Select operators - // i.e. the sequence must be pRS-SEL*-cRS - Operator parent = cRS.getParentOperators().get(0); - while (parent != pRS) { - assert parent.getNumParent() == 1; - if (!(parent instanceof SelectOperator)) { - return false; - } - parent = parent.getParentOperators().get(0); - } - - // If child keys are null or empty, we bail out - if (cKeys == null || cKeys.isEmpty()) { - return false; - } - // If parent keys are null or empty, we bail out - if (pKeys == null || pKeys.isEmpty()) { - return false; - } - - // Backtrack key columns of cRS to pRS - // If we cannot backtrack any of the columns, bail out - List cKeysInParentRS = ExprNodeDescUtils.backtrack(cKeys, cRS, pRS); - for (int i = 0; i < cKeysInParentRS.size(); i++) { - ExprNodeDesc pexpr = cKeysInParentRS.get(i); - if (pexpr == null) { - // We cannot backtrack the expression, we bail out - return false; - } - } - cRS.getConf().setKeyCols(ExprNodeDescUtils.backtrack(cKeysInParentRS, cRS, pRS)); - - // Backtrack partition columns of cRS to pRS - // If we cannot backtrack any of the columns, bail out - List cPartitionInParentRS = ExprNodeDescUtils.backtrack( - cConf.getPartitionCols(), cRS, pRS); - for (int i = 0; i < cPartitionInParentRS.size(); i++) { - ExprNodeDesc pexpr = cPartitionInParentRS.get(i); - if (pexpr == null) { - // We cannot backtrack the expression, we bail out - return false; - } - } - cRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(cPartitionInParentRS, cRS, pRS)); - - // Backtrack value columns of cRS to pRS - // If we cannot backtrack any of the columns, bail out - List cValueInParentRS = ExprNodeDescUtils.backtrack( - cConf.getValueCols(), cRS, pRS); - for (int i = 0; i < cValueInParentRS.size(); i++) { - ExprNodeDesc pexpr = cValueInParentRS.get(i); - if (pexpr == null) { - // We cannot backtrack the expression, we bail out - return false; - } - } - cRS.getConf().setValueCols(ExprNodeDescUtils.backtrack(cValueInParentRS, cRS, pRS)); - - // Backtrack bucket columns of cRS to pRS (if any) - // If we cannot backtrack any of the columns, bail out - if (cConf.getBucketCols() != null) { - List cBucketInParentRS = ExprNodeDescUtils.backtrack( - cConf.getBucketCols(), cRS, pRS); - for (int i = 0; i < cBucketInParentRS.size(); i++) { - ExprNodeDesc pexpr = cBucketInParentRS.get(i); - if (pexpr == null) { - // We cannot backtrack the expression, we bail out - return false; - } - } - cRS.getConf().setBucketCols(ExprNodeDescUtils.backtrack(cBucketInParentRS, cRS, pRS)); - } - - // Update column expression map - for (Entry e : cRS.getColumnExprMap().entrySet()) { - e.setValue(ExprNodeDescUtils.backtrack(e.getValue(), cRS, pRS)); - } - - // Replace pRS with cRS and remove operator sequence from pRS to cRS - // Recall that the sequence must be pRS-SEL*-cRS - parent = cRS.getParentOperators().get(0); - while (parent != pRS) { - dedupCtx.addRemovedOperator(parent); - parent = parent.getParentOperators().get(0); - } - dedupCtx.addRemovedOperator(pRS); - cRS.getParentOperators().clear(); - for (Operator op : pRS.getParentOperators()) { - op.replaceChild(pRS, cRS); - cRS.getParentOperators().add(op); - } - pRS.getParentOperators().clear(); - pRS.getChildOperators().clear(); - - return true; - } } static class GroupbyReducerProc extends AbsctractReducerReducerProc { @@ -652,7 +190,7 @@ public Object process(ReduceSinkOperator cRS, ReduceSinkDeduplicateProcCtx dedup ReduceSinkOperator pRS = CorrelationUtilities.findPossibleParent( pGBY, ReduceSinkOperator.class, dedupCtx.trustScript()); - if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) { + if (pRS != null && ReduceSinkDeDuplicationUtils.merge(cRS, pRS, dedupCtx.minReducer())) { CorrelationUtilities.replaceReduceSinkWithSelectOperator( cRS, dedupCtx.getPctx(), dedupCtx); pRS.getConf().setDeduplicated(true); @@ -675,7 +213,7 @@ public Object process(ReduceSinkOperator cRS, GroupByOperator cGBY, } ReduceSinkOperator pRS = CorrelationUtilities.getSingleParent(pGBY, ReduceSinkOperator.class); - if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) { + if (pRS != null && ReduceSinkDeDuplicationUtils.merge(cRS, pRS, dedupCtx.minReducer())) { CorrelationUtilities.removeReduceSinkForGroupBy( cRS, cGBY, dedupCtx.getPctx(), dedupCtx); pRS.getConf().setDeduplicated(true); @@ -693,7 +231,7 @@ public Object process(ReduceSinkOperator cRS, ReduceSinkDeduplicateProcCtx dedup throws SemanticException { JoinOperator pJoin = CorrelationUtilities.findPossibleParent(cRS, JoinOperator.class, dedupCtx.trustScript()); - if (pJoin != null && merge(cRS, pJoin, dedupCtx.minReducer())) { + if (pJoin != null && ReduceSinkDeDuplicationUtils.merge(cRS, pJoin, dedupCtx.minReducer())) { pJoin.getConf().setFixedAsSorted(true); CorrelationUtilities.replaceReduceSinkWithSelectOperator( cRS, dedupCtx.getPctx(), dedupCtx); @@ -717,7 +255,7 @@ public Object process(ReduceSinkOperator cRS, GroupByOperator cGBY, JoinOperator pJoin = CorrelationUtilities.findPossibleParent( start, JoinOperator.class, dedupCtx.trustScript()); - if (pJoin != null && merge(cRS, pJoin, dedupCtx.minReducer())) { + if (pJoin != null && ReduceSinkDeDuplicationUtils.merge(cRS, pJoin, dedupCtx.minReducer())) { pJoin.getConf().setFixedAsSorted(true); CorrelationUtilities.removeReduceSinkForGroupBy( cRS, cGBY, dedupCtx.getPctx(), dedupCtx); @@ -744,11 +282,11 @@ public Object process(ReduceSinkOperator cRS, ReduceSinkDeduplicateProcCtx dedup cRS, ReduceSinkOperator.class, dedupCtx.trustScript()); if (pRS != null) { // Try extended deduplication - if (aggressiveDedup(cRS, pRS, dedupCtx)) { + if (ReduceSinkDeDuplicationUtils.aggressiveDedup(cRS, pRS, dedupCtx)) { return true; } // Normal deduplication - if (merge(cRS, pRS, dedupCtx.minReducer())) { + if (ReduceSinkDeDuplicationUtils.merge(cRS, pRS, dedupCtx.minReducer())) { CorrelationUtilities.replaceReduceSinkWithSelectOperator( cRS, dedupCtx.getPctx(), dedupCtx); pRS.getConf().setDeduplicated(true); @@ -767,7 +305,7 @@ public Object process(ReduceSinkOperator cRS, GroupByOperator cGBY, ReduceSinkOperator pRS = CorrelationUtilities.findPossibleParent( start, ReduceSinkOperator.class, dedupCtx.trustScript()); - if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) { + if (pRS != null && ReduceSinkDeDuplicationUtils.merge(cRS, pRS, dedupCtx.minReducer())) { if (dedupCtx.getPctx().getConf().getBoolVar(HiveConf.ConfVars.HIVEGROUPBYSKEW)) { return false; } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplicationUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplicationUtils.java new file mode 100644 index 0000000..8f55369 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplicationUtils.java @@ -0,0 +1,584 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.correlation; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.PTFOperator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.SelectOperator; +import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication.ReduceSinkDeduplicateProcCtx; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; +import org.apache.hadoop.hive.ql.plan.TableDesc; + +import com.google.common.collect.ImmutableList; + +public class ReduceSinkDeDuplicationUtils { + + // for JOIN-RS case, it's not possible generally to merge if child has + // less key/partition columns than parents + public static boolean merge(ReduceSinkOperator cRS, JoinOperator pJoin, int minReducer) + throws SemanticException { + List> parents = pJoin.getParentOperators(); + ReduceSinkOperator[] pRSs = parents.toArray(new ReduceSinkOperator[parents.size()]); + ReduceSinkDesc cRSc = cRS.getConf(); + for (ReduceSinkOperator pRSNs : pRSs) { + ReduceSinkDesc pRSNc = pRSNs.getConf(); + if (cRSc.getKeyCols().size() != pRSNc.getKeyCols().size()) { + return false; + } + if (cRSc.getPartitionCols().size() != pRSNc.getPartitionCols().size()) { + return false; + } + Integer moveReducerNumTo = checkNumReducer(cRSc.getNumReducers(), pRSNc.getNumReducers()); + if (moveReducerNumTo == null || + moveReducerNumTo > 0 && cRSc.getNumReducers() < minReducer) { + return false; + } + + Integer moveRSOrderTo = checkOrder(true, cRSc.getOrder(), pRSNc.getOrder(), + cRSc.getNullOrder(), pRSNc.getNullOrder()); + if (moveRSOrderTo == null) { + return false; + } + } + + boolean[] sorted = CorrelationUtilities.getSortedTags(pJoin); + + int cKeySize = cRSc.getKeyCols().size(); + for (int i = 0; i < cKeySize; i++) { + ExprNodeDesc cexpr = cRSc.getKeyCols().get(i); + ExprNodeDesc[] pexprs = new ExprNodeDesc[pRSs.length]; + for (int tag = 0; tag < pRSs.length; tag++) { + pexprs[tag] = pRSs[tag].getConf().getKeyCols().get(i); + } + int found = CorrelationUtilities.indexOf(cexpr, pexprs, cRS, pRSs, sorted); + if (found != i) { + return false; + } + } + int cPartSize = cRSc.getPartitionCols().size(); + for (int i = 0; i < cPartSize; i++) { + ExprNodeDesc cexpr = cRSc.getPartitionCols().get(i); + ExprNodeDesc[] pexprs = new ExprNodeDesc[pRSs.length]; + for (int tag = 0; tag < pRSs.length; tag++) { + pexprs[tag] = pRSs[tag].getConf().getPartitionCols().get(i); + } + int found = CorrelationUtilities.indexOf(cexpr, pexprs, cRS, pRSs, sorted); + if (found != i) { + return false; + } + } + + for (ReduceSinkOperator pRS : pRSs) { + pRS.getConf().setNumReducers(cRS.getConf().getNumReducers()); + } + + return true; + } + + /** + * Current RSDedup remove/replace child RS. For key columns, + * sorting order, and the number of reducers, copy + * more specific part of configurations of child RS to that of parent RS. + * For partitioning columns, if both child RS and parent RS have been assigned + * partitioning columns, we will choose the more general partitioning columns. + * If parent RS has not been assigned any partitioning column, we will use + * partitioning columns (if exist) of child RS. + */ + public static boolean merge(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer) + throws SemanticException { + int[] result = extractMergeDirections(cRS, pRS, minReducer); + if (result == null) { + return false; + } + + if (result[0] > 0) { + // The sorting columns of the child RS are more specific than + // those of the parent RS. Assign sorting columns of the child RS + // to the parent RS. + List childKCs = cRS.getConf().getKeyCols(); + pRS.getConf().setKeyCols(ExprNodeDescUtils.backtrack(childKCs, cRS, pRS)); + } + + if (result[1] < 0) { + // The partitioning columns of the parent RS are more specific than + // those of the child RS. + List childPCs = cRS.getConf().getPartitionCols(); + if (childPCs != null && !childPCs.isEmpty()) { + // If partitioning columns of the child RS are assigned, + // assign these to the partitioning columns of the parent RS. + pRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(childPCs, cRS, pRS)); + } + } else if (result[1] > 0) { + // The partitioning columns of the child RS are more specific than + // those of the parent RS. + List parentPCs = pRS.getConf().getPartitionCols(); + if (parentPCs == null || parentPCs.isEmpty()) { + // If partitioning columns of the parent RS are not assigned, + // assign partitioning columns of the child RS to the parent RS. + ArrayList childPCs = cRS.getConf().getPartitionCols(); + pRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(childPCs, cRS, pRS)); + } + } + + if (result[2] > 0) { + // The sorting order of the child RS is more specific than + // that of the parent RS. Assign the sorting order of the child RS + // to the parent RS. + if (result[0] <= 0) { + // Sorting columns of the parent RS are more specific than those of the + // child RS but Sorting order of the child RS is more specific than + // that of the parent RS. + throw new SemanticException("Sorting columns and order don't match. " + + "Try set " + HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION + "=false;"); + } + pRS.getConf().setOrder(cRS.getConf().getOrder()); + pRS.getConf().setNullOrder(cRS.getConf().getNullOrder()); + } else { + // The sorting order of the parent RS is more specific or they are equal. + // We will copy the order from the child RS, and then fill in the order + // of the rest of columns with the one taken from parent RS. + StringBuilder order = new StringBuilder(cRS.getConf().getOrder()); + StringBuilder orderNull = new StringBuilder(cRS.getConf().getNullOrder()); + order.append(pRS.getConf().getOrder().substring(order.length())); + orderNull.append(pRS.getConf().getNullOrder().substring(orderNull.length())); + pRS.getConf().setOrder(order.toString()); + pRS.getConf().setNullOrder(orderNull.toString()); + } + + if (result[3] > 0) { + // The number of reducers of the child RS is more specific than + // that of the parent RS. Assign the number of reducers of the child RS + // to the parent RS. + pRS.getConf().setNumReducers(cRS.getConf().getNumReducers()); + } + + if (result[4] > 0) { + // This case happens only when pRS key is empty in which case we can use + // number of distribution keys and key serialization info from cRS + if (pRS.getConf().getKeyCols() != null && pRS.getConf().getKeyCols().size() == 0 + && cRS.getConf().getKeyCols() != null && cRS.getConf().getKeyCols().size() == 0) { + // As setNumDistributionKeys is a subset of keycols, the size should + // be 0 too. This condition maybe too strict. We may extend it in the + // future. + TableDesc keyTable = PlanUtils.getReduceKeyTableDesc(new ArrayList(), pRS + .getConf().getOrder(), pRS.getConf().getNullOrder()); + pRS.getConf().setKeySerializeInfo(keyTable); + } + } + return true; + } + + /** + * This is a more strict version of the merge check, where: + * - cRS and pRS should have exactly the same keys in the same positions, and + * - cRS and pRS should have exactly the same partition columns in the same positions, and + * - cRS and pRS should have exactly the same bucket columns in the same positions, and + * - cRS and pRS should sort in the same direction + */ + public static boolean strictMerge(ReduceSinkOperator cRS, ReduceSinkOperator pRS) + throws SemanticException { + return strictMerge(cRS, ImmutableList.of(pRS)); + } + + public static boolean strictMerge(ReduceSinkOperator cRS, List pRSs) + throws SemanticException { + ReduceSinkDesc cRSc = cRS.getConf(); + for (ReduceSinkOperator pRS : pRSs) { + ReduceSinkDesc pRSc = pRS.getConf(); + if (cRSc.getKeyCols().size() != pRSc.getKeyCols().size()) { + return false; + } + if (cRSc.getPartitionCols().size() != pRSc.getPartitionCols().size()) { + return false; + } + + Integer moveRSOrderTo = checkOrder(true, cRSc.getOrder(), pRSc.getOrder(), + cRSc.getNullOrder(), pRSc.getNullOrder()); + if (moveRSOrderTo == null) { + return false; + } + + int cKeySize = cRSc.getKeyCols().size(); + for (int i = 0; i < cKeySize; i++) { + ExprNodeDesc cExpr = cRSc.getKeyCols().get(i); + ExprNodeDesc pExpr = pRSc.getKeyCols().get(i); + if (cExpr instanceof ExprNodeConstantDesc || pExpr instanceof ExprNodeConstantDesc) { + // If ckeys or pkeys have constant node expressions avoid the merge. + return false; + } + ExprNodeDesc backtrackCExpr = ExprNodeDescUtils.backtrack(cExpr, cRS, pRS); + if (backtrackCExpr == null || !pExpr.isSame(backtrackCExpr)) { + return false; + } + } + + int cPartSize = cRSc.getPartitionCols().size(); + for (int i = 0; i < cPartSize; i++) { + ExprNodeDesc cExpr = cRSc.getPartitionCols().get(i); + ExprNodeDesc pExpr = pRSc.getPartitionCols().get(i); + if (cExpr instanceof ExprNodeConstantDesc || pExpr instanceof ExprNodeConstantDesc) { + // If cpartcols or ppartcols have constant node expressions avoid the merge. + return false; + } + ExprNodeDesc backtrackCExpr = ExprNodeDescUtils.backtrack(cExpr, cRS, pRS); + if (backtrackCExpr == null || !pExpr.isSame(backtrackCExpr)) { + return false; + } + } + + if (cRSc.getBucketCols() != null || pRSc.getBucketCols() != null) { + if (cRSc.getBucketCols() == null || pRSc.getBucketCols() == null) { + return false; + } + if (cRSc.getBucketCols().size() != pRSc.getBucketCols().size()) { + return false; + } + int cBucketColsSize = cRSc.getBucketCols().size(); + for (int i = 0; i < cBucketColsSize; i++) { + ExprNodeDesc cExpr = cRSc.getBucketCols().get(i); + ExprNodeDesc pExpr = pRSc.getBucketCols().get(i); + if (cExpr instanceof ExprNodeConstantDesc || pExpr instanceof ExprNodeConstantDesc) { + // If cbucketcols or pbucketcols have constant node expressions avoid the merge. + return false; + } + ExprNodeDesc backtrackCExpr = ExprNodeDescUtils.backtrack(cExpr, cRS, pRS); + if (backtrackCExpr == null || !pExpr.isSame(backtrackCExpr)) { + return false; + } + } + } + + // Meets all requirements + return true; + } + + // Default + return false; + } + + /** + * Returns merge directions between two RSs for criterias (ordering, number of reducers, + * reducer keys, partition keys). Returns null if any of categories is not mergeable. + * + * Values for each index can be -1, 0, 1 + * 1. 0 means two configuration in the category is the same + * 2. for -1, configuration of parent RS is more specific than child RS + * 3. for 1, configuration of child RS is more specific than parent RS + */ + private static int[] extractMergeDirections(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer) + throws SemanticException { + ReduceSinkDesc cConf = cRS.getConf(); + ReduceSinkDesc pConf = pRS.getConf(); + // If there is a PTF between cRS and pRS we cannot ignore the order direction + final boolean checkStrictEquality = isStrictEqualityNeeded(cRS, pRS); + Integer moveRSOrderTo = checkOrder(checkStrictEquality, cConf.getOrder(), pConf.getOrder(), + cConf.getNullOrder(), pConf.getNullOrder()); + if (moveRSOrderTo == null) { + return null; + } + // if cRS is being used for distinct - the two reduce sinks are incompatible + if (cConf.getDistinctColumnIndices().size() >= 2) { + return null; + } + Integer moveReducerNumTo = checkNumReducer(cConf.getNumReducers(), pConf.getNumReducers()); + if (moveReducerNumTo == null || + moveReducerNumTo > 0 && cConf.getNumReducers() < minReducer) { + return null; + } + List ckeys = cConf.getKeyCols(); + List pkeys = pConf.getKeyCols(); + Integer moveKeyColTo = checkExprs(ckeys, pkeys, cRS, pRS); + if (moveKeyColTo == null) { + return null; + } + List cpars = cConf.getPartitionCols(); + List ppars = pConf.getPartitionCols(); + Integer movePartitionColTo = checkExprs(cpars, ppars, cRS, pRS); + if (movePartitionColTo == null) { + return null; + } + Integer moveNumDistKeyTo = checkNumDistributionKey(cConf.getNumDistributionKeys(), + pConf.getNumDistributionKeys()); + return new int[] {moveKeyColTo, movePartitionColTo, moveRSOrderTo, + moveReducerNumTo, moveNumDistKeyTo}; + } + + private static boolean isStrictEqualityNeeded(ReduceSinkOperator cRS, ReduceSinkOperator pRS) { + Operator parent = cRS.getParentOperators().get(0); + while (parent != pRS) { + assert parent.getNumParent() == 1; + if (parent instanceof PTFOperator) { + return true; + } + parent = parent.getParentOperators().get(0); + } + return false; + } + + private static Integer checkNumDistributionKey(int cnd, int pnd) { + // number of distribution keys of cRS is chosen only when numDistKeys of pRS + // is 0 or less. In all other cases, distribution of the keys is based on + // the pRS which is more generic than cRS. + // Examples: + // case 1: if pRS sort key is (a, b) and cRS sort key is (a, b, c) and number of + // distribution keys are 2 and 3 resp. then after merge the sort keys will + // be (a, b, c) while the number of distribution keys will be 2. + // case 2: if pRS sort key is empty and number of distribution keys is 0 + // and if cRS sort key is (a, b) and number of distribution keys is 2 then + // after merge new sort key will be (a, b) and number of distribution keys + // will be 2. + if (pnd <= 0) { + return 1; + } + return 0; + } + + /** + * Overlapping part of keys should be the same between parent and child. + * And if child has more keys than parent, non-overlapping part of keys + * should be backtrackable to parent. + */ + private static Integer checkExprs(List ckeys, List pkeys, + ReduceSinkOperator cRS, ReduceSinkOperator pRS) throws SemanticException { + // If ckeys or pkeys have constant node expressions avoid the merge. + for (ExprNodeDesc ck : ckeys) { + if (ck instanceof ExprNodeConstantDesc) { + return null; + } + } + for (ExprNodeDesc pk : pkeys) { + if (pk instanceof ExprNodeConstantDesc) { + return null; + } + } + + Integer moveKeyColTo = 0; + if (ckeys == null || ckeys.isEmpty()) { + if (pkeys != null && !pkeys.isEmpty()) { + moveKeyColTo = -1; + } + } else { + if (pkeys == null || pkeys.isEmpty()) { + for (ExprNodeDesc ckey : ckeys) { + if (ExprNodeDescUtils.backtrack(ckey, cRS, pRS) == null) { + // cKey is not present in parent + return null; + } + } + moveKeyColTo = 1; + } else { + moveKeyColTo = sameKeys(ckeys, pkeys, cRS, pRS); + } + } + return moveKeyColTo; + } + + // backtrack key exprs of child to parent and compare it with parent's + protected static Integer sameKeys(List cexprs, List pexprs, + Operator child, Operator parent) throws SemanticException { + int common = Math.min(cexprs.size(), pexprs.size()); + int limit = Math.max(cexprs.size(), pexprs.size()); + int i = 0; + for (; i < common; i++) { + ExprNodeDesc pexpr = pexprs.get(i); + ExprNodeDesc cexpr = ExprNodeDescUtils.backtrack(cexprs.get(i), child, parent); + if (cexpr == null || !pexpr.isSame(cexpr)) { + return null; + } + } + for (; i < limit; i++) { + if (cexprs.size() > pexprs.size()) { + if (ExprNodeDescUtils.backtrack(cexprs.get(i), child, parent) == null) { + // cKey is not present in parent + return null; + } + } + } + return Integer.valueOf(cexprs.size()).compareTo(pexprs.size()); + } + + protected static Integer checkOrder(boolean checkStrictEquality, String corder, String porder, + String cNullOrder, String pNullOrder) { + assert corder.length() == cNullOrder.length(); + assert porder.length() == pNullOrder.length(); + if (corder == null || corder.trim().equals("")) { + if (porder == null || porder.trim().equals("")) { + return 0; + } + return -1; + } + if (porder == null || porder.trim().equals("")) { + return 1; + } + corder = corder.trim(); + porder = porder.trim(); + if (checkStrictEquality) { + // order of overlapping keys should be exactly the same + cNullOrder = cNullOrder.trim(); + pNullOrder = pNullOrder.trim(); + int target = Math.min(corder.length(), porder.length()); + if (!corder.substring(0, target).equals(porder.substring(0, target)) || + !cNullOrder.substring(0, target).equals(pNullOrder.substring(0, target))) { + return null; + } + } + return Integer.valueOf(corder.length()).compareTo(porder.length()); + } + + /** + * If number of reducers for RS is -1, the RS can have any number of reducers. + * It's generally true except for order-by or forced bucketing cases. + * if both of num-reducers are not -1, those number should be the same. + */ + protected static Integer checkNumReducer(int creduce, int preduce) { + if (creduce < 0) { + if (preduce < 0) { + return 0; + } + return -1; + } + if (preduce < 0) { + return 1; + } + if (creduce != preduce) { + return null; + } + return 0; + } + + protected static boolean aggressiveDedup(ReduceSinkOperator cRS, ReduceSinkOperator pRS, + ReduceSinkDeduplicateProcCtx dedupCtx) throws SemanticException { + assert cRS.getNumParent() == 1; + + ReduceSinkDesc cConf = cRS.getConf(); + ReduceSinkDesc pConf = pRS.getConf(); + List cKeys = cConf.getKeyCols(); + List pKeys = pConf.getKeyCols(); + + // Check that in the path between cRS and pRS, there are only Select operators + // i.e. the sequence must be pRS-SEL*-cRS + Operator parent = cRS.getParentOperators().get(0); + while (parent != pRS) { + assert parent.getNumParent() == 1; + if (!(parent instanceof SelectOperator)) { + return false; + } + parent = parent.getParentOperators().get(0); + } + + // If child keys are null or empty, we bail out + if (cKeys == null || cKeys.isEmpty()) { + return false; + } + // If parent keys are null or empty, we bail out + if (pKeys == null || pKeys.isEmpty()) { + return false; + } + + // Backtrack key columns of cRS to pRS + // If we cannot backtrack any of the columns, bail out + List cKeysInParentRS = ExprNodeDescUtils.backtrack(cKeys, cRS, pRS); + for (int i = 0; i < cKeysInParentRS.size(); i++) { + ExprNodeDesc pexpr = cKeysInParentRS.get(i); + if (pexpr == null) { + // We cannot backtrack the expression, we bail out + return false; + } + } + cRS.getConf().setKeyCols(ExprNodeDescUtils.backtrack(cKeysInParentRS, cRS, pRS)); + + // Backtrack partition columns of cRS to pRS + // If we cannot backtrack any of the columns, bail out + List cPartitionInParentRS = ExprNodeDescUtils.backtrack( + cConf.getPartitionCols(), cRS, pRS); + for (int i = 0; i < cPartitionInParentRS.size(); i++) { + ExprNodeDesc pexpr = cPartitionInParentRS.get(i); + if (pexpr == null) { + // We cannot backtrack the expression, we bail out + return false; + } + } + cRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(cPartitionInParentRS, cRS, pRS)); + + // Backtrack value columns of cRS to pRS + // If we cannot backtrack any of the columns, bail out + List cValueInParentRS = ExprNodeDescUtils.backtrack( + cConf.getValueCols(), cRS, pRS); + for (int i = 0; i < cValueInParentRS.size(); i++) { + ExprNodeDesc pexpr = cValueInParentRS.get(i); + if (pexpr == null) { + // We cannot backtrack the expression, we bail out + return false; + } + } + cRS.getConf().setValueCols(ExprNodeDescUtils.backtrack(cValueInParentRS, cRS, pRS)); + + // Backtrack bucket columns of cRS to pRS (if any) + // If we cannot backtrack any of the columns, bail out + if (cConf.getBucketCols() != null) { + List cBucketInParentRS = ExprNodeDescUtils.backtrack( + cConf.getBucketCols(), cRS, pRS); + for (int i = 0; i < cBucketInParentRS.size(); i++) { + ExprNodeDesc pexpr = cBucketInParentRS.get(i); + if (pexpr == null) { + // We cannot backtrack the expression, we bail out + return false; + } + } + cRS.getConf().setBucketCols(ExprNodeDescUtils.backtrack(cBucketInParentRS, cRS, pRS)); + } + + // Update column expression map + for (Entry e : cRS.getColumnExprMap().entrySet()) { + e.setValue(ExprNodeDescUtils.backtrack(e.getValue(), cRS, pRS)); + } + + // Replace pRS with cRS and remove operator sequence from pRS to cRS + // Recall that the sequence must be pRS-SEL*-cRS + parent = cRS.getParentOperators().get(0); + while (parent != pRS) { + dedupCtx.addRemovedOperator(parent); + parent = parent.getParentOperators().get(0); + } + dedupCtx.addRemovedOperator(pRS); + cRS.getParentOperators().clear(); + for (Operator op : pRS.getParentOperators()) { + op.replaceChild(pRS, cRS); + cRS.getParentOperators().add(op); + } + pRS.getParentOperators().clear(); + pRS.getChildOperators().clear(); + + return true; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkJoinDeDuplication.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkJoinDeDuplication.java new file mode 100644 index 0000000..d27320b --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkJoinDeDuplication.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.correlation; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Stack; + +import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator; +import org.apache.hadoop.hive.ql.exec.MapJoinOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; +import org.apache.hadoop.hive.ql.lib.Dispatcher; +import org.apache.hadoop.hive.ql.lib.ForwardWalker; +import org.apache.hadoop.hive.ql.lib.GraphWalker; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.lib.Rule; +import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.optimizer.Transform; +import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +/** + * Optimization to check whether any ReduceSink operator in the plan can be + * simplified so data is not shuffled/sorted if it is already shuffled/sorted. + * + * This optimization is executed after join algorithm selection logic has run, + * and it is intended to optimize new cases that cannot be optimized when + * {@link ReduceSinkDeDuplication} runs because some physical algorithms have + * not been selected. Instead of removing ReduceSink operators from the plan, + * they will be tagged, and then the execution plan compiler might take action, + * e.g., on Tez, ReduceSink operators that just need to forward data will be + * translated into a ONE-TO-ONE edge. The parallelism degree of these ReduceSink + * operators might be adjusted, as a ReduceSink operator that just forwards data + * cannot alter the degree of parallelism of the previous task. + */ +public class ReduceSinkJoinDeDuplication extends Transform { + + protected static final Logger LOG = LoggerFactory.getLogger(ReduceSinkJoinDeDuplication.class); + + protected ParseContext pGraphContext; + + @Override + public ParseContext transform(ParseContext pctx) throws SemanticException { + pGraphContext = pctx; + + ReduceSinkJoinDeDuplicateProcCtx cppCtx = new ReduceSinkJoinDeDuplicateProcCtx(pGraphContext); + + Map opRules = new LinkedHashMap(); + opRules.put(new RuleRegExp("R1", ReduceSinkOperator.getOperatorName() + "%"), + ReduceSinkJoinDeDuplicateProcFactory.getReducerMapJoinProc()); + + Dispatcher disp = new DefaultRuleDispatcher( + ReduceSinkJoinDeDuplicateProcFactory.getDefaultProc(), opRules, cppCtx); + GraphWalker ogw = new ForwardWalker(disp); + + // Create a list of topop nodes + ArrayList topNodes = new ArrayList(); + topNodes.addAll(pGraphContext.getTopOps().values()); + ogw.startWalking(topNodes, null); + return pGraphContext; + } + + protected class ReduceSinkJoinDeDuplicateProcCtx extends AbstractCorrelationProcCtx { + + public ReduceSinkJoinDeDuplicateProcCtx(ParseContext pctx) { + super(pctx); + } + } + + static class ReduceSinkJoinDeDuplicateProcFactory { + + public static NodeProcessor getReducerMapJoinProc() { + return new ReducerProc(); + } + + public static NodeProcessor getDefaultProc() { + return new DefaultProc(); + } + } + + /* + * do nothing. + */ + static class DefaultProc implements NodeProcessor { + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + return null; + } + } + + static class ReducerProc implements NodeProcessor { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + ReduceSinkJoinDeDuplicateProcCtx dedupCtx = (ReduceSinkJoinDeDuplicateProcCtx) procCtx; + ReduceSinkOperator cRS = (ReduceSinkOperator) nd; + if (cRS.getConf().isForwarding()) { + // Already set + return false; + } + if (cRS.getConf().getKeyCols().isEmpty()) { + // Not supported + return false; + } + boolean onlyPartitioning = false; + Operator cRSChild = cRS.getChildOperators().get(0); + if (cRSChild instanceof MapJoinOperator || + cRSChild instanceof CommonMergeJoinOperator) { + // If it is a MapJoin or MergeJoin, we make sure that they are on + // the reduce side, otherwise we bail out + for (Operator parent: cRSChild.getParentOperators()) { + if (!(parent instanceof ReduceSinkOperator)) { + // MapJoin and SMBJoin not supported + return false; + } + } + if (cRSChild instanceof MapJoinOperator) { + onlyPartitioning = true; + } + } + + int maxNumReducers = cRS.getConf().getNumReducers(); + ReduceSinkOperator pRS; + if (onlyPartitioning) { + pRS = CorrelationUtilities.findFirstPossibleParent( + cRS, ReduceSinkOperator.class, dedupCtx.trustScript()); + } else { + pRS = CorrelationUtilities.findFirstPossibleParentPreserveSortOrder( + cRS, ReduceSinkOperator.class, dedupCtx.trustScript()); + } + if (pRS != null) { + Operator pRSChild = pRS.getChildOperators().get(0); + if (pRSChild instanceof MapJoinOperator) { + // Handle MapJoin specially and check for all its children + MapJoinOperator pRSChildMJ = (MapJoinOperator) pRSChild; + // In this case, both should be DHJ operators as pRSChildMJ can only guarantee + // partitioned input, not sorted. + if (!pRSChildMJ.getConf().isDynamicPartitionHashJoin() || + !(cRSChild instanceof MapJoinOperator) || + !((MapJoinOperator) cRSChild).getConf().isDynamicPartitionHashJoin()) { + return false; + } + ImmutableList.Builder l = ImmutableList.builder(); + for (Operator parent: pRSChild.getParentOperators()) { + ReduceSinkOperator rsOp = (ReduceSinkOperator) parent; + l.add(rsOp); + if (rsOp.getConf().getNumReducers() > maxNumReducers) { + maxNumReducers = rsOp.getConf().getNumReducers(); + } + } + if (ReduceSinkDeDuplicationUtils.strictMerge(cRS, l.build())) { + LOG.debug("Set {} to forward data", cRS); + cRS.getConf().setForwarding(true); + propagateMaxNumReducers(dedupCtx, cRS, maxNumReducers); + return true; + } + } else if (pRS.getChildOperators().get(0) instanceof CommonMergeJoinOperator) { + // Handle MergeJoin specially and check for all its children + ImmutableList.Builder l = ImmutableList.builder(); + for (Operator parent: pRSChild.getParentOperators()) { + if (!(parent instanceof ReduceSinkOperator)) { + // SMBJoin not supported + return false; + } + ReduceSinkOperator rsOp = (ReduceSinkOperator) parent; + l.add(rsOp); + if (rsOp.getConf().getNumReducers() > maxNumReducers) { + maxNumReducers = rsOp.getConf().getNumReducers(); + } + } + if (ReduceSinkDeDuplicationUtils.strictMerge(cRS, l.build())) { + LOG.debug("Set {} to forward data", cRS); + cRS.getConf().setForwarding(true); + propagateMaxNumReducers(dedupCtx, cRS, maxNumReducers); + return true; + } + } else { + // Rest of cases + if (pRS.getConf().getNumReducers() > maxNumReducers) { + maxNumReducers = pRS.getConf().getNumReducers(); + } + if (ReduceSinkDeDuplicationUtils.strictMerge(cRS, pRS)) { + LOG.debug("Set {} to forward data", cRS); + cRS.getConf().setForwarding(true); + propagateMaxNumReducers(dedupCtx, cRS, maxNumReducers); + return true; + } + } + } + return false; + } + + private static void propagateMaxNumReducers(ReduceSinkJoinDeDuplicateProcCtx dedupCtx, + ReduceSinkOperator rsOp, int maxNumReducers) throws SemanticException { + if (rsOp == null) { + // Bail out + return; + } + if (rsOp.getChildOperators().get(0) instanceof MapJoinOperator || + rsOp.getChildOperators().get(0) instanceof CommonMergeJoinOperator) { + for (Operator p : rsOp.getChildOperators().get(0).getParentOperators()) { + ReduceSinkOperator pRSOp = (ReduceSinkOperator) p; + pRSOp.getConf().setReducerTraits(EnumSet.of(ReducerTraits.FIXED)); + pRSOp.getConf().setNumReducers(maxNumReducers); + LOG.debug("Set {} to FIXED parallelism: {}", pRSOp, maxNumReducers); + if (pRSOp.getConf().isForwarding()) { + ReduceSinkOperator newRSOp = + CorrelationUtilities.findFirstPossibleParent( + pRSOp, ReduceSinkOperator.class, dedupCtx.trustScript()); + propagateMaxNumReducers(dedupCtx, newRSOp, maxNumReducers); + } + } + } else { + rsOp.getConf().setReducerTraits(EnumSet.of(ReducerTraits.FIXED)); + rsOp.getConf().setNumReducers(maxNumReducers); + LOG.debug("Set {} to FIXED parallelism: {}", rsOp, maxNumReducers); + if (rsOp.getConf().isForwarding()) { + ReduceSinkOperator newRSOp = + CorrelationUtilities.findFirstPossibleParent( + rsOp, ReduceSinkOperator.class, dedupCtx.trustScript()); + propagateMaxNumReducers(dedupCtx, newRSOp, maxNumReducers); + } + } + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index 6423a6d..1b0a2f0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -502,6 +502,9 @@ public static void removeBranch(Operator event) { } public static EdgeType determineEdgeType(BaseWork preceedingWork, BaseWork followingWork, ReduceSinkOperator reduceSinkOperator) { + if(reduceSinkOperator.getConf().isForwarding()) { + return EdgeType.ONE_TO_ONE_EDGE; + } if (followingWork instanceof ReduceWork) { // Ideally there should be a better way to determine that the followingWork contains // a dynamic partitioned hash join, but in some cases (createReduceWork()) it looks like diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 888627a..5614c26 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -77,6 +77,7 @@ import org.apache.hadoop.hive.ql.optimizer.RemoveDynamicPruningBySize; import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism; import org.apache.hadoop.hive.ql.optimizer.SharedWorkOptimizer; +import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkJoinDeDuplication; import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits; import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer; import org.apache.hadoop.hive.ql.optimizer.physical.CrossProductCheck; @@ -154,6 +155,16 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, runStatsDependentOptimizations(procCtx, inputs, outputs); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run the optimizations that use stats for optimization"); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + if(procCtx.conf.getBoolVar(ConfVars.HIVEOPTJOINREDUCEDEDUPLICATION)) { + new ReduceSinkJoinDeDuplication().transform(procCtx.parseContext); + } + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run reduce sink after join algorithm selection"); + + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + runRemoveDynamicPruningOptimization(procCtx, inputs, outputs); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run remove dynamic pruning by size"); + // Removing semijoin optimization when it may not be beneficial removeSemijoinOptimizationByBenefit(procCtx); @@ -414,6 +425,25 @@ private void runStatsDependentOptimizations(OptimizeTezProcContext procCtx, opRules.put(new RuleRegExp("Convert Join to Map-join", JoinOperator.getOperatorName() + "%"), new ConvertJoinMapJoin()); + // The dispatcher fires the processor corresponding to the closest matching + // rule and passes the context along + Dispatcher disp = new DefaultRuleDispatcher(null, opRules, procCtx); + List topNodes = new ArrayList(); + topNodes.addAll(procCtx.parseContext.getTopOps().values()); + GraphWalker ogw = new ForwardWalker(disp); + ogw.startWalking(topNodes, null); + } + + private void runRemoveDynamicPruningOptimization(OptimizeTezProcContext procCtx, + Set inputs, Set outputs) throws SemanticException { + + // Sequence of TableScan operators to be walked + Deque> deque = new LinkedList>(); + deque.addAll(procCtx.parseContext.getTopOps().values()); + + // create a walker which walks the tree in a DFS manner while maintaining + // the operator stack. + Map opRules = new LinkedHashMap(); opRules.put( new RuleRegExp("Remove dynamic pruning by size", AppMasterEventOperator.getOperatorName() + "%"), diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java index 24636c5..d08e700 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java @@ -98,6 +98,7 @@ //flag used to control how TopN handled for PTF/Windowing partitions. private boolean isPTFReduceSink = false; private boolean skipTag; // Skip writing tags when feeding into mapjoin hashtable + private boolean forwarding; // Whether this RS can forward records directly instead of shuffling/sorting public static enum ReducerTraits { UNSET(0), // unset @@ -432,6 +433,14 @@ public boolean getSkipTag() { return skipTag; } + public void setForwarding(boolean forwarding) { + this.forwarding = forwarding; + } + + public boolean isForwarding() { + return forwarding; + } + @Explain(displayName = "auto parallelism", explainLevels = { Level.EXTENDED }) public final boolean isAutoParallel() { return (this.reduceTraits.contains(ReducerTraits.AUTOPARALLEL)); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java index a55c708..b695f0f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java @@ -28,6 +28,7 @@ CONTAINS, CUSTOM_EDGE, CUSTOM_SIMPLE_EDGE, + ONE_TO_ONE_EDGE } private HiveConf hiveConf; @@ -102,4 +103,8 @@ public boolean isSlowStart() { public void setSlowStart(boolean slowStart) { this.isSlowStart = slowStart; } + + public void setEdgeType(EdgeType type) { + this.edgeType = type; + } } diff --git ql/src/test/queries/clientpositive/partialdhj.q ql/src/test/queries/clientpositive/partialdhj.q new file mode 100644 index 0000000..c4fe1f7 --- /dev/null +++ ql/src/test/queries/clientpositive/partialdhj.q @@ -0,0 +1,54 @@ +set hive.auto.convert.join=true; +set hive.strict.checks.cartesian.product=false; +set hive.merge.nway.joins=false; +set hive.optimize.shared.work=false; +set hive.optimize.dynamic.partition.hashjoin=true; +set hive.auto.convert.join.hashtable.max.entries=10; + +-- ONE_TO_ONE ON BIG TABLE SIDE +EXPLAIN +SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value); + +SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value); + +-- ONE_TO_ONE ON SMALL TABLE SIDE +EXPLAIN +SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value); + +SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value); diff --git ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out index c563976..8bc280c 100644 --- ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out @@ -322,7 +322,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out index 799de73..ecfa860 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out @@ -22,7 +22,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -190,7 +190,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -358,7 +358,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -524,7 +524,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -690,7 +690,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -856,7 +856,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1022,7 +1022,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1192,7 +1192,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1364,7 +1364,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) @@ -1552,7 +1552,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) @@ -1738,7 +1738,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1919,7 +1919,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out index 611ba06..33aa7ac 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out @@ -22,7 +22,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) @@ -239,7 +239,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) @@ -667,7 +667,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) @@ -884,7 +884,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out index 177cc97..a47a7d3 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out @@ -25,7 +25,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -269,7 +269,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -733,7 +733,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -874,7 +874,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1015,7 +1015,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1196,7 +1196,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1379,7 +1379,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1582,7 +1582,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1785,7 +1785,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2003,7 +2003,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2223,7 +2223,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 4 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2441,7 +2441,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 4 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2660,9 +2660,9 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2914,9 +2914,9 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -3167,7 +3167,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE) @@ -3424,7 +3424,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/explainuser_1.q.out ql/src/test/results/clientpositive/llap/explainuser_1.q.out index eb1268c..b36fe2a 100644 --- ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -1737,7 +1737,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Stage-0 @@ -1752,8 +1752,8 @@ Stage-0 predicate:_col4 is null Merge Join Operator [MERGEJOIN_21] (rows=250 width=178) Conds:RS_13._col0, _col1=RS_14._col0, _col1(Left Outer),Output:["_col0","_col1","_col4"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] + <-Reducer 2 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_13] PartitionCols:_col0, _col1 Group By Operator [GBY_4] (rows=250 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 @@ -1766,8 +1766,8 @@ Stage-0 Output:["key","value"] TableScan [TS_0] (rows=500 width=178) default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] + <-Reducer 4 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_14] PartitionCols:_col0, _col1 Select Operator [SEL_12] (rows=83 width=182) Output:["_col0","_col1","_col2"] @@ -1948,7 +1948,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Stage-0 @@ -1986,8 +1986,8 @@ Stage-0 predicate:((l_linenumber = 1) and l_partkey is not null) TableScan [TS_7] (rows=100 width=16) default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_partkey","l_suppkey","l_linenumber"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] + <-Reducer 2 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_13] PartitionCols:_col0 Group By Operator [GBY_5] (rows=50 width=4) Output:["_col0"],keys:KEY._col0 @@ -2165,7 +2165,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 5 (SIMPLE_EDGE) @@ -2209,8 +2209,8 @@ Stage-0 predicate:(key > '2') TableScan [TS_2] (rows=500 width=87) default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] + <-Reducer 7 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_20] PartitionCols:_col0 Select Operator [SEL_15] (rows=83 width=91) Output:["_col0","_col1"] @@ -2244,8 +2244,8 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) +Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) @@ -2273,8 +2273,8 @@ Stage-0 Output:["_col0","_col1","_col2"] TableScan [TS_0] (rows=26 width=223) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] + <-Reducer 4 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_19] PartitionCols:_col0 Group By Operator [GBY_7] (rows=2 width=114) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 @@ -2288,8 +2288,8 @@ Stage-0 Filter Operator [FIL_29] (rows=8 width=223) predicate:((p_size < 10) and p_mfgr is not null) Please refer to the previous TableScan [TS_0] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] + <-Reducer 5 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_22] PartitionCols:_col0, _col1 Select Operator [SEL_17] (rows=4 width=223) Output:["_col0","_col1","_col2"] @@ -2409,7 +2409,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE) @@ -2451,8 +2451,8 @@ Stage-0 Output:["p_mfgr","p_retailprice"] TableScan [TS_0] (rows=26 width=106) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] + <-Reducer 7 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_28] PartitionCols:_col0 Group By Operator [GBY_16] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/llap/mrr.q.out ql/src/test/results/clientpositive/llap/mrr.q.out index c422021..bfa26e4 100644 --- ql/src/test/results/clientpositive/llap/mrr.q.out +++ ql/src/test/results/clientpositive/llap/mrr.q.out @@ -1293,7 +1293,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE) diff --git ql/src/test/results/clientpositive/llap/parallel.q.out ql/src/test/results/clientpositive/llap/parallel.q.out index 7dba122..aea9417 100644 --- ql/src/test/results/clientpositive/llap/parallel.q.out +++ ql/src/test/results/clientpositive/llap/parallel.q.out @@ -38,7 +38,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/llap/parallel_colstats.q.out ql/src/test/results/clientpositive/llap/parallel_colstats.q.out index 6c95f3a..57498a6 100644 --- ql/src/test/results/clientpositive/llap/parallel_colstats.q.out +++ ql/src/test/results/clientpositive/llap/parallel_colstats.q.out @@ -40,7 +40,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/partialdhj.q.out ql/src/test/results/clientpositive/llap/partialdhj.q.out new file mode 100644 index 0000000..4e62c4f --- /dev/null +++ ql/src/test/results/clientpositive/llap/partialdhj.q.out @@ -0,0 +1,460 @@ +PREHOOK: query: EXPLAIN +SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 KEY.reducesinkkey0 (type: string) + 1 KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Map 4 + Statistics: Num rows: 32 Data size: 2848 Basic stats: COMPLETE Column stats: COMPLETE + HybridGraceHashJoin: true + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 KEY.reducesinkkey0 (type: string) + 1 KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 + input vertices: + 0 Reducer 3 + Statistics: Num rows: 25 Data size: 6675 Basic stats: COMPLETE Column stats: COMPLETE + HybridGraceHashJoin: true + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 6675 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +val_311 311 val_311 +val_165 165 val_165 +val_409 409 val_409 +val_98 98 val_98 +val_484 484 val_484 +val_311 311 val_311 +val_311 311 val_311 +val_165 165 val_165 +val_98 98 val_98 +val_409 409 val_409 +val_409 409 val_409 +val_27 27 val_27 +val_146 146 val_146 +val_146 146 val_146 +val_238 238 val_238 +val_238 238 val_238 +val_193 193 val_193 +val_273 273 val_273 +val_193 193 val_193 +val_273 273 val_273 +val_193 193 val_193 +val_273 273 val_273 +val_406 406 val_406 +val_406 406 val_406 +val_406 406 val_406 +val_406 406 val_406 +val_66 66 val_66 +val_213 213 val_213 +val_213 213 val_213 +val_278 278 val_278 +val_401 401 val_401 +val_278 278 val_278 +val_401 401 val_401 +val_401 401 val_401 +val_401 401 val_401 +val_401 401 val_401 +val_150 150 val_150 +val_255 255 val_255 +val_265 265 val_265 +val_255 255 val_255 +val_265 265 val_265 +PREHOOK: query: EXPLAIN +SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Map 6 (CUSTOM_SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 KEY.reducesinkkey0 (type: string) + 1 KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 + input vertices: + 1 Reducer 5 + Statistics: Num rows: 25 Data size: 6675 Basic stats: COMPLETE Column stats: COMPLETE + HybridGraceHashJoin: true + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 6675 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 KEY.reducesinkkey0 (type: string) + 1 KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Map 6 + Statistics: Num rows: 32 Data size: 2848 Basic stats: COMPLETE Column stats: COMPLETE + HybridGraceHashJoin: true + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +311 val_311 val_311 +165 val_165 val_165 +409 val_409 val_409 +98 val_98 val_98 +484 val_484 val_484 +311 val_311 val_311 +311 val_311 val_311 +165 val_165 val_165 +98 val_98 val_98 +409 val_409 val_409 +409 val_409 val_409 +27 val_27 val_27 +146 val_146 val_146 +146 val_146 val_146 +238 val_238 val_238 +238 val_238 val_238 +193 val_193 val_193 +273 val_273 val_273 +193 val_193 val_193 +273 val_273 val_273 +193 val_193 val_193 +273 val_273 val_273 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +66 val_66 val_66 +213 val_213 val_213 +213 val_213 val_213 +278 val_278 val_278 +401 val_401 val_401 +278 val_278 val_278 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +150 val_150 val_150 +255 val_255 val_255 +265 val_265 val_265 +255 val_255 val_255 +265 val_265 val_265 diff --git ql/src/test/results/clientpositive/llap/subquery_in.q.out ql/src/test/results/clientpositive/llap/subquery_in.q.out index 5c94c6e..53651c4 100644 --- ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -744,7 +744,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -901,7 +901,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1249,7 +1249,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1380,7 +1380,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1523,7 +1523,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1657,7 +1657,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1778,7 +1778,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2082,7 +2082,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2212,7 +2212,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2339,7 +2339,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2730,7 +2730,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) @@ -2955,7 +2955,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) @@ -3111,7 +3111,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) @@ -4115,7 +4115,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) @@ -4578,8 +4578,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Map 5 (SIMPLE_EDGE) @@ -4831,8 +4831,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE) @@ -5591,7 +5591,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 9 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) diff --git ql/src/test/results/clientpositive/llap/subquery_multi.q.out ql/src/test/results/clientpositive/llap/subquery_multi.q.out index bbc8a5b..5673f0e 100644 --- ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -88,7 +88,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -261,9 +261,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Map 5 (CUSTOM_SIMPLE_EDGE) Reducer 8 <- Map 5 (SIMPLE_EDGE) @@ -462,9 +462,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) Reducer 9 <- Map 7 (SIMPLE_EDGE) @@ -696,9 +696,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) Reducer 9 <- Map 7 (SIMPLE_EDGE) @@ -1134,7 +1134,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) @@ -1316,7 +1316,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) @@ -1472,7 +1472,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) @@ -1665,9 +1665,9 @@ STAGE PLANS: Edges: Reducer 10 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 11 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE) @@ -1981,7 +1981,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2156,9 +2156,9 @@ STAGE PLANS: Edges: Reducer 10 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 11 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE) @@ -2472,9 +2472,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE) Reducer 9 <- Map 7 (SIMPLE_EDGE) @@ -2714,7 +2714,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2901,7 +2901,7 @@ STAGE PLANS: Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE) Reducer 11 <- Map 9 (CUSTOM_SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Reducer 10 (CUSTOM_SIMPLE_EDGE) Reducer 8 <- Reducer 11 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) @@ -3873,7 +3873,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -4036,7 +4036,7 @@ STAGE PLANS: Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 10 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) diff --git ql/src/test/results/clientpositive/llap/subquery_notin.q.out ql/src/test/results/clientpositive/llap/subquery_notin.q.out index e4aee8e..31f8f54 100644 --- ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -25,7 +25,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -317,8 +317,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE) @@ -893,8 +893,8 @@ STAGE PLANS: Edges: Reducer 10 <- Map 5 (SIMPLE_EDGE) Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) @@ -1478,7 +1478,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1640,8 +1640,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1841,7 +1841,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 4 (SIMPLE_EDGE) @@ -2056,7 +2056,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 4 (SIMPLE_EDGE) @@ -2279,7 +2279,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) @@ -2505,7 +2505,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 4 (SIMPLE_EDGE) @@ -2705,8 +2705,8 @@ STAGE PLANS: Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) Reducer 11 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) @@ -2975,8 +2975,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -3140,8 +3140,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -3311,11 +3311,11 @@ STAGE PLANS: Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) Reducer 11 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) + Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (ONE_TO_ONE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -3589,7 +3589,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) @@ -3862,8 +3862,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE) @@ -4162,7 +4162,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) @@ -4390,7 +4390,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) @@ -4615,7 +4615,7 @@ STAGE PLANS: Map 8 <- Union 9 (CONTAINS) Reducer 10 <- Union 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 3 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 6 <- Union 5 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -5324,7 +5324,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) @@ -5581,7 +5581,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -5736,8 +5736,8 @@ STAGE PLANS: Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) Reducer 11 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) @@ -6036,8 +6036,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -6251,10 +6251,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 5 <- Map 3 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -6477,10 +6477,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -6667,8 +6667,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -6827,7 +6827,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -6982,7 +6982,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -7145,8 +7145,8 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 10 <- Map 8 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 9 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 10 (CUSTOM_SIMPLE_EDGE) diff --git ql/src/test/results/clientpositive/llap/subquery_scalar.q.out ql/src/test/results/clientpositive/llap/subquery_scalar.q.out index 1556114..06dec8e 100644 --- ql/src/test/results/clientpositive/llap/subquery_scalar.q.out +++ ql/src/test/results/clientpositive/llap/subquery_scalar.q.out @@ -1317,7 +1317,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1447,7 +1447,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1574,7 +1574,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1691,7 +1691,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) @@ -1885,7 +1885,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) @@ -2083,7 +2083,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2263,7 +2263,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -3155,7 +3155,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -3336,7 +3336,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) Reducer 9 <- Map 7 (SIMPLE_EDGE) @@ -3572,8 +3572,8 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 5 (SIMPLE_EDGE) Reducer 9 <- Map 8 (SIMPLE_EDGE) @@ -3800,8 +3800,8 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4009,8 +4009,8 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4211,7 +4211,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -4370,7 +4370,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4673,7 +4673,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4795,7 +4795,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4911,8 +4911,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -5094,7 +5094,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) @@ -5775,7 +5775,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/subquery_select.q.out ql/src/test/results/clientpositive/llap/subquery_select.q.out index eafb77a..fd10898 100644 --- ql/src/test/results/clientpositive/llap/subquery_select.q.out +++ ql/src/test/results/clientpositive/llap/subquery_select.q.out @@ -200,7 +200,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) @@ -586,8 +586,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE) @@ -983,7 +983,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1254,7 +1254,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1391,7 +1391,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1529,7 +1529,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1804,7 +1804,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) @@ -1979,7 +1979,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2154,7 +2154,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2608,7 +2608,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2743,7 +2743,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -3069,7 +3069,7 @@ STAGE PLANS: Edges: Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE) Reducer 11 <- Map 9 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 4 <- Reducer 10 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) @@ -3362,7 +3362,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4812,7 +4812,7 @@ STAGE PLANS: Reducer 11 <- Map 10 (SIMPLE_EDGE) Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 5 <- Reducer 11 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 9 <- Map 8 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -5107,10 +5107,10 @@ STAGE PLANS: Edges: Reducer 10 <- Map 7 (SIMPLE_EDGE) Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 5 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 12 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE) Reducer 9 <- Map 7 (SIMPLE_EDGE) @@ -5478,7 +5478,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/llap/subquery_views.q.out ql/src/test/results/clientpositive/llap/subquery_views.q.out index a2b3fd2..094d6b2 100644 --- ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -124,15 +124,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 13 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 10 <- Reducer 13 (ONE_TO_ONE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 12 <- Map 11 (SIMPLE_EDGE) Reducer 13 <- Map 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) - Reducer 6 <- Reducer 12 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 6 <- Reducer 12 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 8 <- Map 1 (SIMPLE_EDGE) Reducer 9 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out index 6dcf53f..67cd110 100644 --- ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out @@ -151,7 +151,7 @@ STAGE PLANS: Map 5 <- Union 2 (CONTAINS) Map 6 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out index 4b1e92d..b4386c8 100644 --- ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out @@ -413,7 +413,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -547,7 +547,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator + className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE @@ -604,7 +604,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator + className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/perf/query10.q.out ql/src/test/results/clientpositive/perf/query10.q.out index 73c474a..7793d52 100644 --- ql/src/test/results/clientpositive/perf/query10.q.out +++ ql/src/test/results/clientpositive/perf/query10.q.out @@ -125,8 +125,8 @@ Reducer 16 <- Map 13 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) Reducer 17 <- Reducer 16 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 17 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 4 <- Reducer 12 (ONE_TO_ONE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Reducer 17 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) @@ -157,8 +157,8 @@ Stage-0 predicate:(_col16 is not null or _col18 is not null) Merge Join Operator [MERGEJOIN_112] (rows=766650239 width=88) Conds:RS_62._col0=RS_63._col0(Left Outer),Output:["_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col16","_col18"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_63] + <-Reducer 17 [ONE_TO_ONE_EDGE] + FORWARD [RS_63] PartitionCols:_col0 Select Operator [SEL_61] (rows=158394413 width=135) Output:["_col0","_col1"] @@ -196,8 +196,8 @@ Stage-0 Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col16"] Merge Join Operator [MERGEJOIN_111] (rows=696954748 width=88) Conds:RS_43._col0=RS_44._col0(Left Outer),RS_43._col0=RS_45._col0(Inner),Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col15"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_44] + <-Reducer 12 [ONE_TO_ONE_EDGE] + FORWARD [RS_44] PartitionCols:_col0 Select Operator [SEL_22] (rows=79201469 width=135) Output:["_col0","_col1"] @@ -223,8 +223,8 @@ Stage-0 predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null) TableScan [TS_9] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_45] + <-Reducer 15 [ONE_TO_ONE_EDGE] + FORWARD [RS_45] PartitionCols:_col0 Group By Operator [GBY_35] (rows=316797606 width=88) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query14.q.out ql/src/test/results/clientpositive/perf/query14.q.out index b15587c..74c7660 100644 --- ql/src/test/results/clientpositive/perf/query14.q.out +++ ql/src/test/results/clientpositive/perf/query14.q.out @@ -1,6 +1,6 @@ +Warning: Shuffle Join MERGEJOIN[890][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product Warning: Shuffle Join MERGEJOIN[891][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 12' is a cross product Warning: Shuffle Join MERGEJOIN[892][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 16' is a cross product -Warning: Shuffle Join MERGEJOIN[890][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product PREHOOK: query: explain with cross_items as (select i_item_sk ss_item_sk @@ -222,9 +222,9 @@ Reducer 21 <- Map 17 (SIMPLE_EDGE), Map 97 (SIMPLE_EDGE) Reducer 22 <- Map 96 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) Reducer 23 <- Reducer 22 (SIMPLE_EDGE), Union 24 (CONTAINS) Reducer 25 <- Union 24 (SIMPLE_EDGE) -Reducer 26 <- Map 96 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) +Reducer 26 <- Map 96 (SIMPLE_EDGE), Reducer 25 (ONE_TO_ONE_EDGE) Reducer 27 <- Reducer 26 (SIMPLE_EDGE) -Reducer 28 <- Map 96 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE), Reducer 92 (SIMPLE_EDGE) +Reducer 28 <- Map 96 (SIMPLE_EDGE), Reducer 27 (ONE_TO_ONE_EDGE), Reducer 92 (SIMPLE_EDGE) Reducer 29 <- Reducer 28 (SIMPLE_EDGE) Reducer 30 <- Map 17 (SIMPLE_EDGE), Map 98 (SIMPLE_EDGE) Reducer 31 <- Map 96 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) @@ -239,9 +239,9 @@ Reducer 4 <- Union 3 (CUSTOM_SIMPLE_EDGE) Reducer 40 <- Map 96 (SIMPLE_EDGE), Reducer 39 (SIMPLE_EDGE) Reducer 41 <- Reducer 40 (SIMPLE_EDGE), Union 42 (CONTAINS) Reducer 43 <- Union 42 (SIMPLE_EDGE) -Reducer 44 <- Map 96 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) +Reducer 44 <- Map 96 (SIMPLE_EDGE), Reducer 43 (ONE_TO_ONE_EDGE) Reducer 45 <- Reducer 44 (SIMPLE_EDGE) -Reducer 46 <- Map 96 (SIMPLE_EDGE), Reducer 45 (SIMPLE_EDGE), Reducer 94 (SIMPLE_EDGE) +Reducer 46 <- Map 96 (SIMPLE_EDGE), Reducer 45 (ONE_TO_ONE_EDGE), Reducer 94 (SIMPLE_EDGE) Reducer 47 <- Reducer 46 (SIMPLE_EDGE) Reducer 48 <- Map 17 (SIMPLE_EDGE), Map 98 (SIMPLE_EDGE) Reducer 49 <- Map 96 (SIMPLE_EDGE), Reducer 48 (SIMPLE_EDGE) @@ -256,9 +256,9 @@ Reducer 57 <- Map 17 (SIMPLE_EDGE), Map 97 (SIMPLE_EDGE) Reducer 58 <- Map 96 (SIMPLE_EDGE), Reducer 57 (SIMPLE_EDGE) Reducer 59 <- Reducer 58 (SIMPLE_EDGE), Union 60 (CONTAINS) Reducer 61 <- Union 60 (SIMPLE_EDGE) -Reducer 62 <- Map 96 (SIMPLE_EDGE), Reducer 61 (SIMPLE_EDGE) +Reducer 62 <- Map 96 (SIMPLE_EDGE), Reducer 61 (ONE_TO_ONE_EDGE) Reducer 63 <- Reducer 62 (SIMPLE_EDGE) -Reducer 64 <- Map 96 (SIMPLE_EDGE), Reducer 63 (SIMPLE_EDGE), Reducer 95 (SIMPLE_EDGE) +Reducer 64 <- Map 96 (SIMPLE_EDGE), Reducer 63 (ONE_TO_ONE_EDGE), Reducer 95 (SIMPLE_EDGE) Reducer 65 <- Reducer 64 (SIMPLE_EDGE) Reducer 66 <- Map 17 (SIMPLE_EDGE), Map 98 (SIMPLE_EDGE) Reducer 67 <- Map 96 (SIMPLE_EDGE), Reducer 66 (SIMPLE_EDGE) @@ -504,8 +504,8 @@ Stage-0 predicate:i_item_sk is not null TableScan [TS_87] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"] - <-Reducer 45 [SIMPLE_EDGE] - SHUFFLE [RS_362] + <-Reducer 45 [ONE_TO_ONE_EDGE] + FORWARD [RS_362] PartitionCols:_col0 Group By Operator [GBY_355] (rows=254100 width=1436) Output:["_col0"],keys:KEY._col0 @@ -524,8 +524,8 @@ Stage-0 Filter Operator [FIL_810] (rows=462000 width=1436) predicate:(i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_item_sk is not null) Please refer to the previous TableScan [TS_87] - <-Reducer 43 [SIMPLE_EDGE] - SHUFFLE [RS_350] + <-Reducer 43 [ONE_TO_ONE_EDGE] + FORWARD [RS_350] PartitionCols:_col0, _col1, _col2 Select Operator [SEL_348] (rows=1 width=108) Output:["_col0","_col1","_col2"] @@ -833,8 +833,8 @@ Stage-0 Filter Operator [FIL_834] (rows=462000 width=1436) predicate:i_item_sk is not null Please refer to the previous TableScan [TS_87] - <-Reducer 63 [SIMPLE_EDGE] - SHUFFLE [RS_551] + <-Reducer 63 [ONE_TO_ONE_EDGE] + FORWARD [RS_551] PartitionCols:_col0 Group By Operator [GBY_544] (rows=254100 width=1436) Output:["_col0"],keys:KEY._col0 @@ -853,8 +853,8 @@ Stage-0 Filter Operator [FIL_835] (rows=462000 width=1436) predicate:(i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_item_sk is not null) Please refer to the previous TableScan [TS_87] - <-Reducer 61 [SIMPLE_EDGE] - SHUFFLE [RS_539] + <-Reducer 61 [ONE_TO_ONE_EDGE] + FORWARD [RS_539] PartitionCols:_col0, _col1, _col2 Select Operator [SEL_537] (rows=1 width=108) Output:["_col0","_col1","_col2"] @@ -1077,8 +1077,8 @@ Stage-0 Filter Operator [FIL_784] (rows=462000 width=1436) predicate:i_item_sk is not null Please refer to the previous TableScan [TS_87] - <-Reducer 27 [SIMPLE_EDGE] - SHUFFLE [RS_174] + <-Reducer 27 [ONE_TO_ONE_EDGE] + FORWARD [RS_174] PartitionCols:_col0 Group By Operator [GBY_167] (rows=254100 width=1436) Output:["_col0"],keys:KEY._col0 @@ -1097,8 +1097,8 @@ Stage-0 Filter Operator [FIL_785] (rows=462000 width=1436) predicate:(i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_item_sk is not null) Please refer to the previous TableScan [TS_87] - <-Reducer 25 [SIMPLE_EDGE] - SHUFFLE [RS_162] + <-Reducer 25 [ONE_TO_ONE_EDGE] + FORWARD [RS_162] PartitionCols:_col0, _col1, _col2 Select Operator [SEL_160] (rows=1 width=108) Output:["_col0","_col1","_col2"] diff --git ql/src/test/results/clientpositive/perf/query16.q.out ql/src/test/results/clientpositive/perf/query16.q.out index 662bc97..5ae7680 100644 --- ql/src/test/results/clientpositive/perf/query16.q.out +++ ql/src/test/results/clientpositive/perf/query16.q.out @@ -66,7 +66,7 @@ Reducer 15 <- Map 14 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 5 <- Reducer 13 (ONE_TO_ONE_EDGE), Reducer 15 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) @@ -106,8 +106,8 @@ Stage-0 Output:["_col4","_col5","_col6","_col16"] Merge Join Operator [MERGEJOIN_82] (rows=843291907 width=135) Conds:RS_35._col4=RS_36._col0(Left Outer),RS_35._col4=RS_37._col1(Inner),Output:["_col3","_col4","_col5","_col6","_col14","_col15"],residual filter predicates:{(_col3 <> _col15)} - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_36] + <-Reducer 13 [ONE_TO_ONE_EDGE] + FORWARD [RS_36] PartitionCols:_col0 Select Operator [SEL_18] (rows=14399440 width=106) Output:["_col0","_col1"] diff --git ql/src/test/results/clientpositive/perf/query2.q.out ql/src/test/results/clientpositive/perf/query2.q.out index 1686f57..0ba703a 100644 --- ql/src/test/results/clientpositive/perf/query2.q.out +++ ql/src/test/results/clientpositive/perf/query2.q.out @@ -125,10 +125,10 @@ Map 16 <- Union 15 (CONTAINS) Map 8 <- Union 2 (CONTAINS) Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 15 (SIMPLE_EDGE) Reducer 11 <- Reducer 10 (SIMPLE_EDGE) -Reducer 12 <- Map 13 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) +Reducer 12 <- Map 13 (SIMPLE_EDGE), Reducer 11 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 9 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 12 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) @@ -160,8 +160,8 @@ Stage-0 predicate:((d_year = 2002) and d_week_seq is not null) TableScan [TS_20] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_week_seq","d_year"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_50] + <-Reducer 11 [ONE_TO_ONE_EDGE] + FORWARD [RS_50] PartitionCols:_col0 Group By Operator [GBY_45] (rows=237595882 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)"],keys:KEY._col0 @@ -215,8 +215,8 @@ Stage-0 Filter Operator [FIL_86] (rows=36524 width=1119) predicate:((d_year = 2001) and d_week_seq is not null) Please refer to the previous TableScan [TS_20] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_23] + <-Reducer 4 [ONE_TO_ONE_EDGE] + FORWARD [RS_23] PartitionCols:_col0 Group By Operator [GBY_18] (rows=237595882 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query23.q.out ql/src/test/results/clientpositive/perf/query23.q.out index 0e34b90..ebd2271 100644 --- ql/src/test/results/clientpositive/perf/query23.q.out +++ ql/src/test/results/clientpositive/perf/query23.q.out @@ -1,5 +1,5 @@ -Warning: Shuffle Join MERGEJOIN[369][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 30' is a cross product Warning: Shuffle Join MERGEJOIN[367][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 25' is a cross product +Warning: Shuffle Join MERGEJOIN[369][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 30' is a cross product PREHOOK: query: explain with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt diff --git ql/src/test/results/clientpositive/perf/query31.q.out ql/src/test/results/clientpositive/perf/query31.q.out index 79b9e52..644d3e1 100644 --- ql/src/test/results/clientpositive/perf/query31.q.out +++ ql/src/test/results/clientpositive/perf/query31.q.out @@ -108,7 +108,7 @@ Reducer 11 <- Reducer 10 (SIMPLE_EDGE) Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) Reducer 14 <- Map 23 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE) Reducer 15 <- Reducer 14 (SIMPLE_EDGE) -Reducer 16 <- Reducer 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) +Reducer 16 <- Reducer 15 (ONE_TO_ONE_EDGE), Reducer 19 (ONE_TO_ONE_EDGE), Reducer 22 (ONE_TO_ONE_EDGE) Reducer 17 <- Map 12 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE) Reducer 18 <- Map 23 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) Reducer 19 <- Reducer 18 (SIMPLE_EDGE) @@ -118,7 +118,7 @@ Reducer 21 <- Map 23 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) Reducer 22 <- Reducer 21 (SIMPLE_EDGE) Reducer 3 <- Map 23 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 11 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 5 <- Reducer 11 (ONE_TO_ONE_EDGE), Reducer 16 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE) Reducer 7 <- Map 23 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) @@ -136,8 +136,8 @@ Stage-0 predicate:(CASE WHEN ((_col1 > 0)) THEN (CASE WHEN ((_col9 > 0)) THEN (((_col11 / _col9) > (_col5 / _col1))) ELSE ((null > (_col5 / _col1))) END) ELSE (CASE WHEN ((_col9 > 0)) THEN (((_col11 / _col9) > null)) ELSE (null) END) END and CASE WHEN ((_col3 > 0)) THEN (CASE WHEN ((_col7 > 0)) THEN (((_col9 / _col7) > (_col1 / _col3))) ELSE ((null > (_col1 / _col3))) END) ELSE (CASE WHEN ((_col7 > 0)) THEN (((_col9 / _col7) > null)) ELSE (null) END) END) Merge Join Operator [MERGEJOIN_267] (rows=1149975359 width=88) Conds:RS_125._col0=RS_126._col0(Inner),RS_125._col0=RS_127._col0(Inner),RS_125._col0=RS_128._col0(Inner),Output:["_col0","_col1","_col3","_col5","_col7","_col9","_col11"] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_127] + <-Reducer 11 [ONE_TO_ONE_EDGE] + FORWARD [RS_127] PartitionCols:_col0 Group By Operator [GBY_58] (rows=348477374 width=88) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 @@ -185,8 +185,8 @@ Stage-0 PartitionCols:_col0 Merge Join Operator [MERGEJOIN_266] (rows=191667561 width=135) Conds:RS_120._col0=RS_121._col0(Inner),RS_120._col0=RS_122._col0(Inner),Output:["_col0","_col1","_col3","_col5"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_120] + <-Reducer 15 [ONE_TO_ONE_EDGE] + FORWARD [RS_120] PartitionCols:_col0 Group By Operator [GBY_78] (rows=87121617 width=135) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 @@ -223,8 +223,8 @@ Stage-0 predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null) TableScan [TS_60] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_addr_sk","ws_ext_sales_price"] - <-Reducer 19 [SIMPLE_EDGE] - SHUFFLE [RS_121] + <-Reducer 19 [ONE_TO_ONE_EDGE] + FORWARD [RS_121] PartitionCols:_col0 Group By Operator [GBY_98] (rows=87121617 width=135) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 @@ -256,8 +256,8 @@ Stage-0 SHUFFLE [RS_89] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_62] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_122] + <-Reducer 22 [ONE_TO_ONE_EDGE] + FORWARD [RS_122] PartitionCols:_col0 Group By Operator [GBY_118] (rows=87121617 width=135) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 @@ -289,8 +289,8 @@ Stage-0 SHUFFLE [RS_109] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_62] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_125] + <-Reducer 4 [ONE_TO_ONE_EDGE] + FORWARD [RS_125] PartitionCols:_col0 Group By Operator [GBY_18] (rows=348477374 width=88) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 @@ -322,8 +322,8 @@ Stage-0 SHUFFLE [RS_9] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_2] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_126] + <-Reducer 8 [ONE_TO_ONE_EDGE] + FORWARD [RS_126] PartitionCols:_col0 Group By Operator [GBY_38] (rows=348477374 width=88) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query32.q.out ql/src/test/results/clientpositive/perf/query32.q.out index 10ad060..7b21854 100644 --- ql/src/test/results/clientpositive/perf/query32.q.out +++ ql/src/test/results/clientpositive/perf/query32.q.out @@ -60,7 +60,7 @@ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 7 <- Map 9 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) +Reducer 7 <- Map 9 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Stage-0 Fetch Operator @@ -119,8 +119,8 @@ Stage-0 predicate:((i_manufact_id = 269) and i_item_sk is not null) TableScan [TS_20] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 6 [SIMPLE_EDGE] - SHUFFLE [RS_23] + <-Reducer 6 [ONE_TO_ONE_EDGE] + FORWARD [RS_23] PartitionCols:_col1 Select Operator [SEL_19] (rows=158394413 width=135) Output:["_col0","_col1"] diff --git ql/src/test/results/clientpositive/perf/query33.q.out ql/src/test/results/clientpositive/perf/query33.q.out index aab3a19..9d47c8a 100644 --- ql/src/test/results/clientpositive/perf/query33.q.out +++ ql/src/test/results/clientpositive/perf/query33.q.out @@ -150,14 +150,14 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) +Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (ONE_TO_ONE_EDGE) Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 15 <- Map 14 (SIMPLE_EDGE) Reducer 16 <- Map 14 (SIMPLE_EDGE) Reducer 17 <- Map 14 (SIMPLE_EDGE) Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE) Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE) Reducer 23 <- Map 26 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) @@ -167,7 +167,7 @@ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) +Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (ONE_TO_ONE_EDGE) Reducer 9 <- Reducer 23 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Stage-0 @@ -252,8 +252,8 @@ Stage-0 predicate:(i_manufact_id is not null and i_item_sk is not null) TableScan [TS_0] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_64] + <-Reducer 16 [ONE_TO_ONE_EDGE] + FORWARD [RS_64] PartitionCols:_col0 Group By Operator [GBY_45] (rows=115500 width=1436) Output:["_col0"],keys:KEY._col0 @@ -291,8 +291,8 @@ Stage-0 SHUFFLE [RS_101] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_2] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_102] + <-Reducer 17 [ONE_TO_ONE_EDGE] + FORWARD [RS_102] PartitionCols:_col0 Group By Operator [GBY_83] (rows=115500 width=1436) Output:["_col0"],keys:KEY._col0 @@ -352,8 +352,8 @@ Stage-0 SHUFFLE [RS_26] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_2] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_27] + <-Reducer 15 [ONE_TO_ONE_EDGE] + FORWARD [RS_27] PartitionCols:_col0 Group By Operator [GBY_8] (rows=115500 width=1436) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query35.q.out ql/src/test/results/clientpositive/perf/query35.q.out index 2815257..5d45b17 100644 --- ql/src/test/results/clientpositive/perf/query35.q.out +++ ql/src/test/results/clientpositive/perf/query35.q.out @@ -121,8 +121,8 @@ Reducer 16 <- Map 13 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) Reducer 17 <- Reducer 16 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 17 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 4 <- Reducer 12 (ONE_TO_ONE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Reducer 17 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) @@ -153,8 +153,8 @@ Stage-0 predicate:(_col13 is not null or _col15 is not null) Merge Join Operator [MERGEJOIN_113] (rows=766650239 width=88) Conds:RS_62._col0=RS_63._col0(Left Outer),Output:["_col4","_col6","_col7","_col8","_col9","_col10","_col13","_col15"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_63] + <-Reducer 17 [ONE_TO_ONE_EDGE] + FORWARD [RS_63] PartitionCols:_col0 Select Operator [SEL_61] (rows=158394413 width=135) Output:["_col0","_col1"] @@ -192,8 +192,8 @@ Stage-0 Output:["_col0","_col4","_col6","_col7","_col8","_col9","_col10","_col13"] Merge Join Operator [MERGEJOIN_112] (rows=696954748 width=88) Conds:RS_43._col0=RS_44._col0(Left Outer),RS_43._col0=RS_45._col0(Inner),Output:["_col0","_col4","_col6","_col7","_col8","_col9","_col10","_col12"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_44] + <-Reducer 12 [ONE_TO_ONE_EDGE] + FORWARD [RS_44] PartitionCols:_col0 Select Operator [SEL_22] (rows=79201469 width=135) Output:["_col0","_col1"] @@ -219,8 +219,8 @@ Stage-0 predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null) TableScan [TS_9] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_45] + <-Reducer 15 [ONE_TO_ONE_EDGE] + FORWARD [RS_45] PartitionCols:_col0 Group By Operator [GBY_35] (rows=316797606 width=88) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query45.q.out ql/src/test/results/clientpositive/perf/query45.q.out index 19cf365..112b47b 100644 --- ql/src/test/results/clientpositive/perf/query45.q.out +++ ql/src/test/results/clientpositive/perf/query45.q.out @@ -49,7 +49,7 @@ Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 12 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) -Reducer 9 <- Map 8 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) +Reducer 9 <- Map 8 (SIMPLE_EDGE), Reducer 11 (ONE_TO_ONE_EDGE) Stage-0 Fetch Operator @@ -137,8 +137,8 @@ Stage-0 Filter Operator [FIL_77] (rows=462000 width=1436) predicate:i_item_sk is not null Please refer to the previous TableScan [TS_6] - <-Reducer 11 [SIMPLE_EDGE] - SHUFFLE [RS_27] + <-Reducer 11 [ONE_TO_ONE_EDGE] + FORWARD [RS_27] PartitionCols:_col0 Select Operator [SEL_15] (rows=115500 width=1436) Output:["_col0","_col1"] diff --git ql/src/test/results/clientpositive/perf/query56.q.out ql/src/test/results/clientpositive/perf/query56.q.out index 8f4606a..395a53b 100644 --- ql/src/test/results/clientpositive/perf/query56.q.out +++ ql/src/test/results/clientpositive/perf/query56.q.out @@ -136,14 +136,14 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) +Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (ONE_TO_ONE_EDGE) Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 15 <- Map 14 (SIMPLE_EDGE) Reducer 16 <- Map 14 (SIMPLE_EDGE) Reducer 17 <- Map 14 (SIMPLE_EDGE) Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE) Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE) Reducer 23 <- Map 26 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) @@ -153,7 +153,7 @@ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) +Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (ONE_TO_ONE_EDGE) Reducer 9 <- Reducer 23 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Stage-0 @@ -238,8 +238,8 @@ Stage-0 predicate:(i_item_id is not null and i_item_sk is not null) TableScan [TS_0] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_64] + <-Reducer 16 [ONE_TO_ONE_EDGE] + FORWARD [RS_64] PartitionCols:_col0 Group By Operator [GBY_45] (rows=115500 width=1436) Output:["_col0"],keys:KEY._col0 @@ -277,8 +277,8 @@ Stage-0 SHUFFLE [RS_101] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_2] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_102] + <-Reducer 17 [ONE_TO_ONE_EDGE] + FORWARD [RS_102] PartitionCols:_col0 Group By Operator [GBY_83] (rows=115500 width=1436) Output:["_col0"],keys:KEY._col0 @@ -338,8 +338,8 @@ Stage-0 SHUFFLE [RS_26] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_2] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_27] + <-Reducer 15 [ONE_TO_ONE_EDGE] + FORWARD [RS_27] PartitionCols:_col0 Group By Operator [GBY_8] (rows=115500 width=1436) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query58.q.out ql/src/test/results/clientpositive/perf/query58.q.out index 5b37f36..7be1ff5 100644 --- ql/src/test/results/clientpositive/perf/query58.q.out +++ ql/src/test/results/clientpositive/perf/query58.q.out @@ -136,9 +136,9 @@ Reducer 10 <- Reducer 9 (SIMPLE_EDGE) Reducer 11 <- Map 33 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 22 (ONE_TO_ONE_EDGE) +Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 26 (ONE_TO_ONE_EDGE) +Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 30 (ONE_TO_ONE_EDGE) Reducer 19 <- Map 18 (CUSTOM_SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 20 <- Map 31 (CUSTOM_SIMPLE_EDGE), Reducer 19 (CUSTOM_SIMPLE_EDGE) @@ -154,7 +154,7 @@ Reducer 29 <- Map 31 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE) Reducer 3 <- Reducer 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 30 <- Reducer 29 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 5 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 8 <- Map 32 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 16 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) @@ -177,8 +177,8 @@ Stage-0 predicate:(_col5 BETWEEN (0.9 * _col1) AND (1.1 * _col1) and _col5 BETWEEN (0.9 * _col3) AND (1.1 * _col3) and _col1 BETWEEN (0.9 * _col5) AND (1.1 * _col5) and _col3 BETWEEN (0.9 * _col5) AND (1.1 * _col5) and _col1 BETWEEN (0.9 * _col3) AND (1.1 * _col3) and _col3 BETWEEN (0.9 * _col1) AND (1.1 * _col1)) Merge Join Operator [MERGEJOIN_279] (rows=766650239 width=88) Conds:RS_150._col0=RS_151._col0(Inner),RS_150._col0=RS_152._col0(Inner),Output:["_col0","_col1","_col3","_col5"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_151] + <-Reducer 10 [ONE_TO_ONE_EDGE] + FORWARD [RS_151] PartitionCols:_col0 Group By Operator [GBY_98] (rows=348477374 width=88) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 @@ -203,8 +203,8 @@ Stage-0 predicate:(d_date is not null and d_date_sk is not null) TableScan [TS_6] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_86] + <-Reducer 26 [ONE_TO_ONE_EDGE] + FORWARD [RS_86] PartitionCols:_col0 Group By Operator [GBY_83] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 @@ -275,8 +275,8 @@ Stage-0 predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) TableScan [TS_50] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ext_sales_price"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_152] + <-Reducer 13 [ONE_TO_ONE_EDGE] + FORWARD [RS_152] PartitionCols:_col0 Group By Operator [GBY_148] (rows=87121617 width=135) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 @@ -314,8 +314,8 @@ Stage-0 SHUFFLE [RS_135] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_8] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_136] + <-Reducer 30 [ONE_TO_ONE_EDGE] + FORWARD [RS_136] PartitionCols:_col0 Group By Operator [GBY_133] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 @@ -356,8 +356,8 @@ Stage-0 <-Map 18 [CUSTOM_SIMPLE_EDGE] PARTITION_ONLY_SHUFFLE [RS_113] Please refer to the previous Group By Operator [GBY_12] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_150] + <-Reducer 4 [ONE_TO_ONE_EDGE] + FORWARD [RS_150] PartitionCols:_col0 Group By Operator [GBY_48] (rows=174233858 width=135) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 @@ -377,8 +377,8 @@ Stage-0 SHUFFLE [RS_35] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_8] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_36] + <-Reducer 22 [ONE_TO_ONE_EDGE] + FORWARD [RS_36] PartitionCols:_col0 Group By Operator [GBY_33] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query6.q.out ql/src/test/results/clientpositive/perf/query6.q.out index 6ea20e9..1c685e2 100644 --- ql/src/test/results/clientpositive/perf/query6.q.out +++ ql/src/test/results/clientpositive/perf/query6.q.out @@ -58,7 +58,7 @@ Reducer 12 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 19 (CUSTOM_SIMPLE_EDGE) Reducer 13 <- Map 20 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) Reducer 19 <- Map 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 16 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) @@ -208,8 +208,8 @@ Stage-0 predicate:(d_date_sk is not null and d_month_seq is not null) TableScan [TS_0] (rows=73049 width=1119) default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_56] + <-Reducer 9 [ONE_TO_ONE_EDGE] + FORWARD [RS_56] PartitionCols:_col0 Group By Operator [GBY_8] (rows=9131 width=1119) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query60.q.out ql/src/test/results/clientpositive/perf/query60.q.out index 7d63986..13974bc 100644 --- ql/src/test/results/clientpositive/perf/query60.q.out +++ ql/src/test/results/clientpositive/perf/query60.q.out @@ -156,14 +156,14 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS) -Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) +Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (ONE_TO_ONE_EDGE) Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 15 <- Map 14 (SIMPLE_EDGE) Reducer 16 <- Map 14 (SIMPLE_EDGE) Reducer 17 <- Map 14 (SIMPLE_EDGE) Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE) Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE) Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE) Reducer 23 <- Map 26 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) @@ -173,7 +173,7 @@ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS) Reducer 6 <- Union 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE) +Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (ONE_TO_ONE_EDGE) Reducer 9 <- Reducer 23 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Stage-0 @@ -258,8 +258,8 @@ Stage-0 predicate:(i_item_id is not null and i_item_sk is not null) TableScan [TS_0] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"] - <-Reducer 16 [SIMPLE_EDGE] - SHUFFLE [RS_64] + <-Reducer 16 [ONE_TO_ONE_EDGE] + FORWARD [RS_64] PartitionCols:_col0 Group By Operator [GBY_45] (rows=115500 width=1436) Output:["_col0"],keys:KEY._col0 @@ -297,8 +297,8 @@ Stage-0 SHUFFLE [RS_101] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_2] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_102] + <-Reducer 17 [ONE_TO_ONE_EDGE] + FORWARD [RS_102] PartitionCols:_col0 Group By Operator [GBY_83] (rows=115500 width=1436) Output:["_col0"],keys:KEY._col0 @@ -358,8 +358,8 @@ Stage-0 SHUFFLE [RS_26] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_2] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_27] + <-Reducer 15 [ONE_TO_ONE_EDGE] + FORWARD [RS_27] PartitionCols:_col0 Group By Operator [GBY_8] (rows=115500 width=1436) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query64.q.out ql/src/test/results/clientpositive/perf/query64.q.out index ddd0614..b31fd7f 100644 --- ql/src/test/results/clientpositive/perf/query64.q.out +++ ql/src/test/results/clientpositive/perf/query64.q.out @@ -251,14 +251,14 @@ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE) Reducer 21 <- Map 20 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE) Reducer 22 <- Map 48 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE) Reducer 23 <- Reducer 22 (SIMPLE_EDGE), Reducer 35 (SIMPLE_EDGE) -Reducer 24 <- Reducer 23 (SIMPLE_EDGE), Reducer 51 (SIMPLE_EDGE) +Reducer 24 <- Reducer 23 (SIMPLE_EDGE), Reducer 51 (ONE_TO_ONE_EDGE) Reducer 25 <- Map 55 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) Reducer 26 <- Map 39 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE) Reducer 27 <- Map 20 (SIMPLE_EDGE), Reducer 45 (SIMPLE_EDGE) Reducer 28 <- Map 48 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE) Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Reducer 37 (SIMPLE_EDGE) Reducer 3 <- Map 20 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 30 <- Reducer 29 (SIMPLE_EDGE), Reducer 53 (SIMPLE_EDGE) +Reducer 30 <- Reducer 29 (SIMPLE_EDGE), Reducer 53 (ONE_TO_ONE_EDGE) Reducer 31 <- Map 55 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) Reducer 32 <- Map 39 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE) Reducer 34 <- Map 33 (SIMPLE_EDGE), Map 38 (SIMPLE_EDGE) @@ -529,8 +529,8 @@ Stage-0 SHUFFLE [RS_210] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_14] - <-Reducer 53 [SIMPLE_EDGE] - SHUFFLE [RS_216] + <-Reducer 53 [ONE_TO_ONE_EDGE] + FORWARD [RS_216] PartitionCols:_col0 Select Operator [SEL_202] (rows=52798137 width=135) Output:["_col0"] @@ -681,8 +681,8 @@ Stage-0 SHUFFLE [RS_82] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_14] - <-Reducer 51 [SIMPLE_EDGE] - SHUFFLE [RS_88] + <-Reducer 51 [ONE_TO_ONE_EDGE] + FORWARD [RS_88] PartitionCols:_col0 Select Operator [SEL_74] (rows=52798137 width=135) Output:["_col0"] diff --git ql/src/test/results/clientpositive/perf/query69.q.out ql/src/test/results/clientpositive/perf/query69.q.out index 2769bc5..1228894 100644 --- ql/src/test/results/clientpositive/perf/query69.q.out +++ ql/src/test/results/clientpositive/perf/query69.q.out @@ -101,8 +101,8 @@ Reducer 16 <- Map 13 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) Reducer 17 <- Reducer 16 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 17 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 4 <- Reducer 12 (ONE_TO_ONE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) +Reducer 5 <- Reducer 17 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) @@ -133,8 +133,8 @@ Stage-0 predicate:_col15 is null Merge Join Operator [MERGEJOIN_114] (rows=383325119 width=88) Conds:RS_64._col0=RS_65._col0(Left Outer),Output:["_col6","_col7","_col8","_col9","_col10","_col15"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_65] + <-Reducer 17 [ONE_TO_ONE_EDGE] + FORWARD [RS_65] PartitionCols:_col0 Select Operator [SEL_63] (rows=158394413 width=135) Output:["_col0","_col1"] @@ -176,8 +176,8 @@ Stage-0 Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col13"] Merge Join Operator [MERGEJOIN_113] (rows=696954748 width=88) Conds:RS_43._col0=RS_44._col0(Left Outer),RS_43._col0=RS_45._col0(Inner),Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col12"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_44] + <-Reducer 12 [ONE_TO_ONE_EDGE] + FORWARD [RS_44] PartitionCols:_col0 Select Operator [SEL_22] (rows=79201469 width=135) Output:["_col0","_col1"] @@ -203,8 +203,8 @@ Stage-0 predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null) TableScan [TS_9] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"] - <-Reducer 15 [SIMPLE_EDGE] - SHUFFLE [RS_45] + <-Reducer 15 [ONE_TO_ONE_EDGE] + FORWARD [RS_45] PartitionCols:_col0 Group By Operator [GBY_35] (rows=316797606 width=88) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query77.q.out ql/src/test/results/clientpositive/perf/query77.q.out index db42b18..d10e226 100644 --- ql/src/test/results/clientpositive/perf/query77.q.out +++ ql/src/test/results/clientpositive/perf/query77.q.out @@ -226,13 +226,13 @@ Reducer 18 <- Map 29 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 19 <- Map 30 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 20 <- Reducer 19 (SIMPLE_EDGE) -Reducer 21 <- Reducer 20 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 21 <- Reducer 20 (ONE_TO_ONE_EDGE), Reducer 24 (ONE_TO_ONE_EDGE), Union 6 (CONTAINS) Reducer 22 <- Map 31 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 23 <- Map 30 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) Reducer 24 <- Reducer 23 (SIMPLE_EDGE) Reducer 3 <- Map 25 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS) +Reducer 5 <- Reducer 12 (ONE_TO_ONE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Union 6 (CONTAINS) Reducer 7 <- Union 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) @@ -323,8 +323,8 @@ Stage-0 Output:["_col0","_col1","_col2","_col3","_col4"] Merge Join Operator [MERGEJOIN_189] (rows=95833780 width=135) Conds:RS_117._col0=RS_118._col0(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5"] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_117] + <-Reducer 20 [ONE_TO_ONE_EDGE] + FORWARD [RS_117] PartitionCols:_col0 Group By Operator [GBY_95] (rows=87121617 width=135) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 @@ -362,8 +362,8 @@ Stage-0 predicate:(ws_sold_date_sk is not null and ws_web_page_sk is not null) TableScan [TS_77] (rows=144002668 width=135) default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_web_page_sk","ws_ext_sales_price","ws_net_profit"] - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_118] + <-Reducer 24 [ONE_TO_ONE_EDGE] + FORWARD [RS_118] PartitionCols:_col0 Group By Operator [GBY_115] (rows=8711072 width=92) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 @@ -405,8 +405,8 @@ Stage-0 Output:["_col0","_col1","_col2","_col3","_col4"] Merge Join Operator [MERGEJOIN_187] (rows=383325119 width=88) Conds:RS_40._col0=RS_41._col0(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5"] - <-Reducer 12 [SIMPLE_EDGE] - SHUFFLE [RS_41] + <-Reducer 12 [ONE_TO_ONE_EDGE] + FORWARD [RS_41] PartitionCols:_col0 Group By Operator [GBY_38] (rows=34842647 width=77) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 @@ -444,8 +444,8 @@ Stage-0 predicate:(sr_returned_date_sk is not null and sr_store_sk is not null) TableScan [TS_20] (rows=57591150 width=77) default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_store_sk","sr_return_amt","sr_net_loss"] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_40] + <-Reducer 4 [ONE_TO_ONE_EDGE] + FORWARD [RS_40] PartitionCols:_col0 Group By Operator [GBY_18] (rows=348477374 width=88) Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query78.q.out ql/src/test/results/clientpositive/perf/query78.q.out index 2fccac2..91c244f 100644 --- ql/src/test/results/clientpositive/perf/query78.q.out +++ ql/src/test/results/clientpositive/perf/query78.q.out @@ -121,7 +121,7 @@ Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE) @@ -203,8 +203,8 @@ Stage-0 predicate:(COALESCE(_col7,0) > 0) Merge Join Operator [MERGEJOIN_112] (rows=191662559 width=88) Conds:RS_42._col1, _col0=RS_43._col1, _col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col7","_col8","_col9"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_42] + <-Reducer 3 [ONE_TO_ONE_EDGE] + FORWARD [RS_42] PartitionCols:_col1, _col0 Select Operator [SEL_20] (rows=174238687 width=88) Output:["_col0","_col1","_col2","_col3","_col4"] @@ -246,8 +246,8 @@ Stage-0 Output:["_col0","_col1"] TableScan [TS_6] (rows=57591150 width=77) default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_43] + <-Reducer 8 [ONE_TO_ONE_EDGE] + FORWARD [RS_43] PartitionCols:_col1, _col0 Select Operator [SEL_41] (rows=43560808 width=135) Output:["_col0","_col1","_col2","_col3","_col4"] diff --git ql/src/test/results/clientpositive/perf/query83.q.out ql/src/test/results/clientpositive/perf/query83.q.out index 4f14cc6..dfae770 100644 --- ql/src/test/results/clientpositive/perf/query83.q.out +++ ql/src/test/results/clientpositive/perf/query83.q.out @@ -137,9 +137,9 @@ Reducer 10 <- Reducer 9 (SIMPLE_EDGE) Reducer 11 <- Map 27 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 20 (ONE_TO_ONE_EDGE) +Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 22 (ONE_TO_ONE_EDGE) +Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 24 (ONE_TO_ONE_EDGE) Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 25 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 20 <- Reducer 19 (SIMPLE_EDGE) @@ -149,7 +149,7 @@ Reducer 23 <- Map 18 (SIMPLE_EDGE), Map 25 (SIMPLE_EDGE) Reducer 24 <- Reducer 23 (SIMPLE_EDGE) Reducer 3 <- Reducer 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 5 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 8 <- Map 26 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 16 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) @@ -170,8 +170,8 @@ Stage-0 Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] Merge Join Operator [MERGEJOIN_222] (rows=76653825 width=77) Conds:RS_120._col0=RS_121._col0(Inner),RS_120._col0=RS_122._col0(Inner),Output:["_col0","_col1","_col3","_col5"] - <-Reducer 10 [SIMPLE_EDGE] - SHUFFLE [RS_121] + <-Reducer 10 [ONE_TO_ONE_EDGE] + FORWARD [RS_121] PartitionCols:_col0 Group By Operator [GBY_78] (rows=34842647 width=77) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 @@ -196,8 +196,8 @@ Stage-0 predicate:(d_date is not null and d_date_sk is not null) TableScan [TS_6] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_66] + <-Reducer 22 [ONE_TO_ONE_EDGE] + FORWARD [RS_66] PartitionCols:_col0 Group By Operator [GBY_63] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 @@ -251,8 +251,8 @@ Stage-0 predicate:(sr_item_sk is not null and sr_returned_date_sk is not null) TableScan [TS_40] (rows=57591150 width=77) default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_return_quantity"] - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_122] + <-Reducer 13 [ONE_TO_ONE_EDGE] + FORWARD [RS_122] PartitionCols:_col0 Group By Operator [GBY_118] (rows=8711072 width=92) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 @@ -290,8 +290,8 @@ Stage-0 SHUFFLE [RS_105] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_8] - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_106] + <-Reducer 24 [ONE_TO_ONE_EDGE] + FORWARD [RS_106] PartitionCols:_col0 Group By Operator [GBY_103] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 @@ -310,8 +310,8 @@ Stage-0 SHUFFLE [RS_98] PartitionCols:_col0 Please refer to the previous Group By Operator [GBY_16] - <-Reducer 4 [SIMPLE_EDGE] - SHUFFLE [RS_120] + <-Reducer 4 [ONE_TO_ONE_EDGE] + FORWARD [RS_120] PartitionCols:_col0 Group By Operator [GBY_38] (rows=17423323 width=106) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 @@ -331,8 +331,8 @@ Stage-0 SHUFFLE [RS_25] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_8] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_26] + <-Reducer 20 [ONE_TO_ONE_EDGE] + FORWARD [RS_26] PartitionCols:_col0 Group By Operator [GBY_23] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query92.q.out ql/src/test/results/clientpositive/perf/query92.q.out index 761753f..adcbd833 100644 --- ql/src/test/results/clientpositive/perf/query92.q.out +++ ql/src/test/results/clientpositive/perf/query92.q.out @@ -65,7 +65,7 @@ Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) -Reducer 8 <- Map 10 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Reducer 8 <- Map 10 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Stage-0 Fetch Operator @@ -130,8 +130,8 @@ Stage-0 predicate:((i_manufact_id = 269) and i_item_sk is not null) TableScan [TS_20] (rows=462000 width=1436) default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"] - <-Reducer 7 [SIMPLE_EDGE] - SHUFFLE [RS_23] + <-Reducer 7 [ONE_TO_ONE_EDGE] + FORWARD [RS_23] PartitionCols:_col1 Select Operator [SEL_19] (rows=79201469 width=135) Output:["_col0","_col1"] diff --git ql/src/test/results/clientpositive/perf/query94.q.out ql/src/test/results/clientpositive/perf/query94.q.out index 77fba1b..b76fed3 100644 --- ql/src/test/results/clientpositive/perf/query94.q.out +++ ql/src/test/results/clientpositive/perf/query94.q.out @@ -62,7 +62,7 @@ Reducer 15 <- Map 14 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 5 <- Reducer 13 (ONE_TO_ONE_EDGE), Reducer 15 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) @@ -102,8 +102,8 @@ Stage-0 Output:["_col4","_col5","_col6","_col16"] Merge Join Operator [MERGEJOIN_82] (rows=421668645 width=135) Conds:RS_35._col4=RS_36._col0(Left Outer),RS_35._col4=RS_37._col1(Inner),Output:["_col3","_col4","_col5","_col6","_col14","_col15"],residual filter predicates:{(_col3 <> _col15)} - <-Reducer 13 [SIMPLE_EDGE] - SHUFFLE [RS_36] + <-Reducer 13 [ONE_TO_ONE_EDGE] + FORWARD [RS_36] PartitionCols:_col0 Select Operator [SEL_18] (rows=7199233 width=92) Output:["_col0","_col1"] diff --git ql/src/test/results/clientpositive/perf/query95.q.out ql/src/test/results/clientpositive/perf/query95.q.out index 28db5a6..f036a47 100644 --- ql/src/test/results/clientpositive/perf/query95.q.out +++ ql/src/test/results/clientpositive/perf/query95.q.out @@ -71,7 +71,7 @@ Reducer 17 <- Reducer 16 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) -Reducer 5 <- Reducer 14 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 5 <- Reducer 14 (ONE_TO_ONE_EDGE), Reducer 17 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) @@ -105,8 +105,8 @@ Stage-0 Output:["_col0","_col2","_col3"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col3 Merge Join Operator [MERGEJOIN_127] (rows=421668645 width=135) Conds:RS_58._col3=RS_59._col0(Inner),RS_58._col3=RS_60._col0(Inner),Output:["_col3","_col4","_col5"] - <-Reducer 14 [SIMPLE_EDGE] - SHUFFLE [RS_59] + <-Reducer 14 [ONE_TO_ONE_EDGE] + FORWARD [RS_59] PartitionCols:_col0 Group By Operator [GBY_25] (rows=79201469 width=135) Output:["_col0"],keys:KEY._col0 @@ -139,8 +139,8 @@ Stage-0 predicate:ws_order_number is not null TableScan [TS_15] (rows=144002668 width=135) default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"] - <-Reducer 17 [SIMPLE_EDGE] - SHUFFLE [RS_60] + <-Reducer 17 [ONE_TO_ONE_EDGE] + FORWARD [RS_60] PartitionCols:_col0 Group By Operator [GBY_47] (rows=87121617 width=135) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query97.q.out ql/src/test/results/clientpositive/perf/query97.q.out index 51dd341..81589f7 100644 --- ql/src/test/results/clientpositive/perf/query97.q.out +++ ql/src/test/results/clientpositive/perf/query97.q.out @@ -51,7 +51,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) -Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) +Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) @@ -74,8 +74,8 @@ Stage-0 Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_49] (rows=348477374 width=88) Conds:RS_28._col0, _col1=RS_29._col0, _col1(Outer),Output:["_col0","_col2"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_28] + <-Reducer 3 [ONE_TO_ONE_EDGE] + FORWARD [RS_28] PartitionCols:_col0, _col1 Group By Operator [GBY_12] (rows=316797606 width=88) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 @@ -104,8 +104,8 @@ Stage-0 predicate:ss_sold_date_sk is not null TableScan [TS_0] (rows=575995635 width=88) default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk"] - <-Reducer 8 [SIMPLE_EDGE] - SHUFFLE [RS_29] + <-Reducer 8 [ONE_TO_ONE_EDGE] + FORWARD [RS_29] PartitionCols:_col0, _col1 Group By Operator [GBY_26] (rows=158394413 width=135) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1