diff --git common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java index 799355a..d754be8 100644 --- common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java +++ common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java @@ -74,7 +74,7 @@ public VertexType vertexType; public static enum EdgeType { - BROADCAST, SHUFFLE, MULTICAST, PARTITION_ONLY_SHUFFLE, UNKNOWN + BROADCAST, SHUFFLE, MULTICAST, PARTITION_ONLY_SHUFFLE, ONE_TO_ONE, UNKNOWN }; public String edgeType; diff --git common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java index 294dc6b..69e5358 100644 --- common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java +++ common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/TezJsonParser.java @@ -34,6 +34,8 @@ public String mapEdgeType(String edgeName) { return "PARTITION_ONLY_SHUFFLE"; case "CUSTOM_EDGE": return "MULTICAST"; + case "ONE_TO_ONE_EDGE": + return "FORWARD"; default: return "UNKNOWN"; } diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 9c954be..c8d8b46 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1601,6 +1601,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Reduce deduplication merges two RSs by moving key/parts/reducer-num of the child RS to parent RS. \n" + "That means if reducer-num of the child RS is fixed (order by or forced bucketing) and small, it can make very slow, single MR.\n" + "The optimization will be automatically disabled if number of reducers would be less than specified value."), + HIVEOPTJOINREDUCEDEDUPLICATION("hive.optimize.joinreducededuplication", true, + "Remove extra shuffle/sorting operations after join algorithm selection has been executed. \n" + + "Currently it only works with Apache Tez. This should always be set to true. \n" + + "Since it is a new feature, it has been made configurable."), HIVEOPTSORTDYNAMICPARTITION("hive.optimize.sort.dynamic.partition", false, "When enabled dynamic partitioning column will be globally sorted.\n" + diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 1cc0104..362a796 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -742,7 +742,8 @@ minillaplocal.query.files=acid_globallimit.q,\ groupby_resolution.q,\ windowing_windowspec2.q,\ vectorized_join46.q,\ - vectorized_multi_output_select.q + vectorized_multi_output_select.q,\ + partialdhj.q encrypted.query.files=encryption_join_unencrypted_tbl.q,\ encryption_insert_partition_static.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 51beaad..64e0d9f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -293,6 +293,10 @@ public GroupInputEdge createEdge(VertexGroup group, JobConf vConf, Vertex w, mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; + case ONE_TO_ONE_EDGE: + mergeInputClass = ConcatenatedMergedKeyValueInput.class; + break; + case SIMPLE_EDGE: setupAutoReducerParallelism(edgeProp, w); // fall through @@ -398,18 +402,26 @@ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp, Configuration .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); return et3Conf.createDefaultEdgeProperty(); + case ONE_TO_ONE_EDGE: + UnorderedKVEdgeConfig et4Conf = UnorderedKVEdgeConfig + .newBuilder(keyClass, valClass) + .setFromConfiguration(conf) + .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null) + .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) + .build(); + return et4Conf.createDefaultOneToOneEdgeProperty(); case SIMPLE_EDGE: default: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); - OrderedPartitionedKVEdgeConfig et4Conf = OrderedPartitionedKVEdgeConfig + OrderedPartitionedKVEdgeConfig et5Conf = OrderedPartitionedKVEdgeConfig .newBuilder(keyClass, valClass, MRPartitioner.class.getName(), partitionerConf) .setFromConfiguration(conf) .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), TezBytesComparator.class.getName(), null) .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); - return et4Conf.createDefaultEdgeProperty(); + return et5Conf.createDefaultEdgeProperty(); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java index d84a1e6..0538176 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java @@ -1073,8 +1073,11 @@ public static MapJoinDesc getMapJoinDesc(HiveConf hconf, ExprNodeDesc expr = colExprMap.get(column.getInternalName()); int index = ExprNodeDescUtils.indexOf(expr, values); if (index >= 0) { - colExprMap.put(column.getInternalName(), newValues.get(index)); schema.set(i, null); + if (adjustParentsChildren) { + // Since we remove reduce sink parents, replace original expressions + colExprMap.put(column.getInternalName(), newValues.get(index)); + } } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java index ac234d0..70919a6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java @@ -270,7 +270,11 @@ public static Object processReduceSinkToHashJoin(ReduceSinkOperator parentRS, Ma } } } else if (mapJoinOp.getConf().isDynamicPartitionHashJoin()) { - edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + if (parentRS.getConf().isForwarding()) { + edgeType = EdgeType.ONE_TO_ONE_EDGE; + } else { + edgeType = EdgeType.CUSTOM_SIMPLE_EDGE; + } } if (edgeType == EdgeType.CUSTOM_EDGE) { // disable auto parallelism for bucket map joins diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java index 388399c..c63c28d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java @@ -29,8 +29,8 @@ import java.util.Map.Entry; import java.util.Set; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.CommonJoinOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.ForwardOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; @@ -43,16 +43,15 @@ import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.Utilities.ReduceField; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication.ReduceSinkDeduplicateProcCtx; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AggregationDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.GroupByDesc; +import org.apache.hadoop.hive.ql.plan.GroupByDesc.Mode; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -271,6 +270,52 @@ protected static int indexOf(ExprNodeDesc cexpr, ExprNodeDesc[] pexprs, Operator return result; } + protected static > T findFirstPossibleParent( + Operator start, Class target, boolean trustScript) throws SemanticException { + // Preserve only partitioning + return findFirstPossibleParent(start, target, trustScript, false); + } + + protected static > T findFirstPossibleParentPreserveSortOrder( + Operator start, Class target, boolean trustScript) throws SemanticException { + // Preserve partitioning and ordering + return findFirstPossibleParent(start, target, trustScript, true); + } + + @SuppressWarnings("unchecked") + private static > T findFirstPossibleParent( + Operator start, Class target, boolean trustScript, boolean preserveSortOrder) + throws SemanticException { + Operator cursor = CorrelationUtilities.getSingleParent(start); + for (; cursor != null; cursor = CorrelationUtilities.getSingleParent(cursor)) { + if (target.isAssignableFrom(cursor.getClass())) { + return (T) cursor; + } + if (cursor instanceof CommonJoinOperator) { + for (Operator op : ((CommonJoinOperator) cursor).getParentOperators()) { + if (target.isAssignableFrom(op.getClass())) { + return (T) op; + } + } + return null; + } + if (cursor instanceof ScriptOperator && !trustScript) { + return null; + } + if (!(cursor instanceof SelectOperator + || cursor instanceof FilterOperator + || cursor instanceof ForwardOperator + || cursor instanceof ScriptOperator + || (cursor instanceof GroupByOperator + && (!preserveSortOrder + || ((GroupByOperator) cursor).getConf().getMode() != Mode.HASH)) // Not order preserving + || cursor instanceof ReduceSinkOperator)) { + return null; + } + } + return null; + } + /** * Search the query plan tree from startPoint to the bottom. If there is no ReduceSinkOperator * between startPoint and the corresponding TableScanOperator, return the corresponding diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java index 701bde4..154c607 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java @@ -23,18 +23,14 @@ import java.util.ArrayList; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Stack; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.PTFOperator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; @@ -49,13 +45,6 @@ import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.PlanUtils; -import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.TableDesc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,457 +173,6 @@ protected abstract Object process(ReduceSinkOperator cRS, ReduceSinkDeduplicateP protected abstract Object process(ReduceSinkOperator cRS, GroupByOperator cGBY, ReduceSinkDeduplicateProcCtx dedupCtx) throws SemanticException; - - // for JOIN-RS case, it's not possible generally to merge if child has - // less key/partition columns than parents - protected boolean merge(ReduceSinkOperator cRS, JoinOperator pJoin, int minReducer) - throws SemanticException { - List> parents = pJoin.getParentOperators(); - ReduceSinkOperator[] pRSs = parents.toArray(new ReduceSinkOperator[parents.size()]); - ReduceSinkDesc cRSc = cRS.getConf(); - for (ReduceSinkOperator pRSNs : pRSs) { - ReduceSinkDesc pRSNc = pRSNs.getConf(); - if (cRSc.getKeyCols().size() != pRSNc.getKeyCols().size()) { - return false; - } - if (cRSc.getPartitionCols().size() != pRSNc.getPartitionCols().size()) { - return false; - } - Integer moveReducerNumTo = checkNumReducer(cRSc.getNumReducers(), pRSNc.getNumReducers()); - if (moveReducerNumTo == null || - moveReducerNumTo > 0 && cRSc.getNumReducers() < minReducer) { - return false; - } - - Integer moveRSOrderTo = checkOrder(true, cRSc.getOrder(), pRSNc.getOrder(), - cRSc.getNullOrder(), pRSNc.getNullOrder()); - if (moveRSOrderTo == null) { - return false; - } - } - - boolean[] sorted = CorrelationUtilities.getSortedTags(pJoin); - - int cKeySize = cRSc.getKeyCols().size(); - for (int i = 0; i < cKeySize; i++) { - ExprNodeDesc cexpr = cRSc.getKeyCols().get(i); - ExprNodeDesc[] pexprs = new ExprNodeDesc[pRSs.length]; - for (int tag = 0; tag < pRSs.length; tag++) { - pexprs[tag] = pRSs[tag].getConf().getKeyCols().get(i); - } - int found = CorrelationUtilities.indexOf(cexpr, pexprs, cRS, pRSs, sorted); - if (found != i) { - return false; - } - } - int cPartSize = cRSc.getPartitionCols().size(); - for (int i = 0; i < cPartSize; i++) { - ExprNodeDesc cexpr = cRSc.getPartitionCols().get(i); - ExprNodeDesc[] pexprs = new ExprNodeDesc[pRSs.length]; - for (int tag = 0; tag < pRSs.length; tag++) { - pexprs[tag] = pRSs[tag].getConf().getPartitionCols().get(i); - } - int found = CorrelationUtilities.indexOf(cexpr, pexprs, cRS, pRSs, sorted); - if (found != i) { - return false; - } - } - - for (ReduceSinkOperator pRS : pRSs) { - pRS.getConf().setNumReducers(cRS.getConf().getNumReducers()); - } - - return true; - } - - /** - * Current RSDedup remove/replace child RS. For key columns, - * sorting order, and the number of reducers, copy - * more specific part of configurations of child RS to that of parent RS. - * For partitioning columns, if both child RS and parent RS have been assigned - * partitioning columns, we will choose the more general partitioning columns. - * If parent RS has not been assigned any partitioning column, we will use - * partitioning columns (if exist) of child RS. - */ - protected boolean merge(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer) - throws SemanticException { - int[] result = extractMergeDirections(cRS, pRS, minReducer); - if (result == null) { - return false; - } - - if (result[0] > 0) { - // The sorting columns of the child RS are more specific than - // those of the parent RS. Assign sorting columns of the child RS - // to the parent RS. - List childKCs = cRS.getConf().getKeyCols(); - pRS.getConf().setKeyCols(ExprNodeDescUtils.backtrack(childKCs, cRS, pRS)); - } - - if (result[1] < 0) { - // The partitioning columns of the parent RS are more specific than - // those of the child RS. - List childPCs = cRS.getConf().getPartitionCols(); - if (childPCs != null && !childPCs.isEmpty()) { - // If partitioning columns of the child RS are assigned, - // assign these to the partitioning columns of the parent RS. - pRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(childPCs, cRS, pRS)); - } - } else if (result[1] > 0) { - // The partitioning columns of the child RS are more specific than - // those of the parent RS. - List parentPCs = pRS.getConf().getPartitionCols(); - if (parentPCs == null || parentPCs.isEmpty()) { - // If partitioning columns of the parent RS are not assigned, - // assign partitioning columns of the child RS to the parent RS. - ArrayList childPCs = cRS.getConf().getPartitionCols(); - pRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(childPCs, cRS, pRS)); - } - } - - if (result[2] > 0) { - // The sorting order of the child RS is more specific than - // that of the parent RS. Assign the sorting order of the child RS - // to the parent RS. - if (result[0] <= 0) { - // Sorting columns of the parent RS are more specific than those of the - // child RS but Sorting order of the child RS is more specific than - // that of the parent RS. - throw new SemanticException("Sorting columns and order don't match. " + - "Try set " + HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION + "=false;"); - } - pRS.getConf().setOrder(cRS.getConf().getOrder()); - pRS.getConf().setNullOrder(cRS.getConf().getNullOrder()); - } else { - // The sorting order of the parent RS is more specific or they are equal. - // We will copy the order from the child RS, and then fill in the order - // of the rest of columns with the one taken from parent RS. - StringBuilder order = new StringBuilder(cRS.getConf().getOrder()); - StringBuilder orderNull = new StringBuilder(cRS.getConf().getNullOrder()); - order.append(pRS.getConf().getOrder().substring(order.length())); - orderNull.append(pRS.getConf().getNullOrder().substring(orderNull.length())); - pRS.getConf().setOrder(order.toString()); - pRS.getConf().setNullOrder(orderNull.toString()); - } - - if (result[3] > 0) { - // The number of reducers of the child RS is more specific than - // that of the parent RS. Assign the number of reducers of the child RS - // to the parent RS. - pRS.getConf().setNumReducers(cRS.getConf().getNumReducers()); - } - - if (result[4] > 0) { - // This case happens only when pRS key is empty in which case we can use - // number of distribution keys and key serialization info from cRS - if (pRS.getConf().getKeyCols() != null && pRS.getConf().getKeyCols().size() == 0 - && cRS.getConf().getKeyCols() != null && cRS.getConf().getKeyCols().size() == 0) { - // As setNumDistributionKeys is a subset of keycols, the size should - // be 0 too. This condition maybe too strict. We may extend it in the - // future. - TableDesc keyTable = PlanUtils.getReduceKeyTableDesc(new ArrayList(), pRS - .getConf().getOrder(), pRS.getConf().getNullOrder()); - pRS.getConf().setKeySerializeInfo(keyTable); - } - } - return true; - } - - /** - * Returns merge directions between two RSs for criterias (ordering, number of reducers, - * reducer keys, partition keys). Returns null if any of categories is not mergeable. - * - * Values for each index can be -1, 0, 1 - * 1. 0 means two configuration in the category is the same - * 2. for -1, configuration of parent RS is more specific than child RS - * 3. for 1, configuration of child RS is more specific than parent RS - */ - private int[] extractMergeDirections(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer) - throws SemanticException { - ReduceSinkDesc cConf = cRS.getConf(); - ReduceSinkDesc pConf = pRS.getConf(); - // If there is a PTF between cRS and pRS we cannot ignore the order direction - final boolean checkStrictEquality = isStrictEqualityNeeded(cRS, pRS); - Integer moveRSOrderTo = checkOrder(checkStrictEquality, cConf.getOrder(), pConf.getOrder(), - cConf.getNullOrder(), pConf.getNullOrder()); - if (moveRSOrderTo == null) { - return null; - } - // if cRS is being used for distinct - the two reduce sinks are incompatible - if (cConf.getDistinctColumnIndices().size() >= 2) { - return null; - } - Integer moveReducerNumTo = checkNumReducer(cConf.getNumReducers(), pConf.getNumReducers()); - if (moveReducerNumTo == null || - moveReducerNumTo > 0 && cConf.getNumReducers() < minReducer) { - return null; - } - List ckeys = cConf.getKeyCols(); - List pkeys = pConf.getKeyCols(); - Integer moveKeyColTo = checkExprs(ckeys, pkeys, cRS, pRS); - if (moveKeyColTo == null) { - return null; - } - List cpars = cConf.getPartitionCols(); - List ppars = pConf.getPartitionCols(); - Integer movePartitionColTo = checkExprs(cpars, ppars, cRS, pRS); - if (movePartitionColTo == null) { - return null; - } - Integer moveNumDistKeyTo = checkNumDistributionKey(cConf.getNumDistributionKeys(), - pConf.getNumDistributionKeys()); - return new int[] {moveKeyColTo, movePartitionColTo, moveRSOrderTo, - moveReducerNumTo, moveNumDistKeyTo}; - } - - private boolean isStrictEqualityNeeded(ReduceSinkOperator cRS, ReduceSinkOperator pRS) { - Operator parent = cRS.getParentOperators().get(0); - while (parent != pRS) { - assert parent.getNumParent() == 1; - if (parent instanceof PTFOperator) { - return true; - } - parent = parent.getParentOperators().get(0); - } - return false; - } - - private Integer checkNumDistributionKey(int cnd, int pnd) { - // number of distribution keys of cRS is chosen only when numDistKeys of pRS - // is 0 or less. In all other cases, distribution of the keys is based on - // the pRS which is more generic than cRS. - // Examples: - // case 1: if pRS sort key is (a, b) and cRS sort key is (a, b, c) and number of - // distribution keys are 2 and 3 resp. then after merge the sort keys will - // be (a, b, c) while the number of distribution keys will be 2. - // case 2: if pRS sort key is empty and number of distribution keys is 0 - // and if cRS sort key is (a, b) and number of distribution keys is 2 then - // after merge new sort key will be (a, b) and number of distribution keys - // will be 2. - if (pnd <= 0) { - return 1; - } - return 0; - } - - /** - * Overlapping part of keys should be the same between parent and child. - * And if child has more keys than parent, non-overlapping part of keys - * should be backtrackable to parent. - */ - private Integer checkExprs(List ckeys, List pkeys, - ReduceSinkOperator cRS, ReduceSinkOperator pRS) throws SemanticException { - // If ckeys or pkeys have constant node expressions avoid the merge. - for (ExprNodeDesc ck : ckeys) { - if (ck instanceof ExprNodeConstantDesc) { - return null; - } - } - for (ExprNodeDesc pk : pkeys) { - if (pk instanceof ExprNodeConstantDesc) { - return null; - } - } - - Integer moveKeyColTo = 0; - if (ckeys == null || ckeys.isEmpty()) { - if (pkeys != null && !pkeys.isEmpty()) { - moveKeyColTo = -1; - } - } else { - if (pkeys == null || pkeys.isEmpty()) { - for (ExprNodeDesc ckey : ckeys) { - if (ExprNodeDescUtils.backtrack(ckey, cRS, pRS) == null) { - // cKey is not present in parent - return null; - } - } - moveKeyColTo = 1; - } else { - moveKeyColTo = sameKeys(ckeys, pkeys, cRS, pRS); - } - } - return moveKeyColTo; - } - - // backtrack key exprs of child to parent and compare it with parent's - protected Integer sameKeys(List cexprs, List pexprs, - Operator child, Operator parent) throws SemanticException { - int common = Math.min(cexprs.size(), pexprs.size()); - int limit = Math.max(cexprs.size(), pexprs.size()); - int i = 0; - for (; i < common; i++) { - ExprNodeDesc pexpr = pexprs.get(i); - ExprNodeDesc cexpr = ExprNodeDescUtils.backtrack(cexprs.get(i), child, parent); - if (cexpr == null || !pexpr.isSame(cexpr)) { - return null; - } - } - for (; i < limit; i++) { - if (cexprs.size() > pexprs.size()) { - if (ExprNodeDescUtils.backtrack(cexprs.get(i), child, parent) == null) { - // cKey is not present in parent - return null; - } - } - } - return Integer.valueOf(cexprs.size()).compareTo(pexprs.size()); - } - - protected Integer checkOrder(boolean checkStrictEquality, String corder, String porder, - String cNullOrder, String pNullOrder) { - assert corder.length() == cNullOrder.length(); - assert porder.length() == pNullOrder.length(); - if (corder == null || corder.trim().equals("")) { - if (porder == null || porder.trim().equals("")) { - return 0; - } - return -1; - } - if (porder == null || porder.trim().equals("")) { - return 1; - } - corder = corder.trim(); - porder = porder.trim(); - if (checkStrictEquality) { - // order of overlapping keys should be exactly the same - cNullOrder = cNullOrder.trim(); - pNullOrder = pNullOrder.trim(); - int target = Math.min(corder.length(), porder.length()); - if (!corder.substring(0, target).equals(porder.substring(0, target)) || - !cNullOrder.substring(0, target).equals(pNullOrder.substring(0, target))) { - return null; - } - } - return Integer.valueOf(corder.length()).compareTo(porder.length()); - } - - /** - * If number of reducers for RS is -1, the RS can have any number of reducers. - * It's generally true except for order-by or forced bucketing cases. - * if both of num-reducers are not -1, those number should be the same. - */ - protected Integer checkNumReducer(int creduce, int preduce) { - if (creduce < 0) { - if (preduce < 0) { - return 0; - } - return -1; - } - if (preduce < 0) { - return 1; - } - if (creduce != preduce) { - return null; - } - return 0; - } - - protected boolean aggressiveDedup(ReduceSinkOperator cRS, ReduceSinkOperator pRS, - ReduceSinkDeduplicateProcCtx dedupCtx) throws SemanticException { - assert cRS.getNumParent() == 1; - - ReduceSinkDesc cConf = cRS.getConf(); - ReduceSinkDesc pConf = pRS.getConf(); - List cKeys = cConf.getKeyCols(); - List pKeys = pConf.getKeyCols(); - - // Check that in the path between cRS and pRS, there are only Select operators - // i.e. the sequence must be pRS-SEL*-cRS - Operator parent = cRS.getParentOperators().get(0); - while (parent != pRS) { - assert parent.getNumParent() == 1; - if (!(parent instanceof SelectOperator)) { - return false; - } - parent = parent.getParentOperators().get(0); - } - - // If child keys are null or empty, we bail out - if (cKeys == null || cKeys.isEmpty()) { - return false; - } - // If parent keys are null or empty, we bail out - if (pKeys == null || pKeys.isEmpty()) { - return false; - } - - // Backtrack key columns of cRS to pRS - // If we cannot backtrack any of the columns, bail out - List cKeysInParentRS = ExprNodeDescUtils.backtrack(cKeys, cRS, pRS); - for (int i = 0; i < cKeysInParentRS.size(); i++) { - ExprNodeDesc pexpr = cKeysInParentRS.get(i); - if (pexpr == null) { - // We cannot backtrack the expression, we bail out - return false; - } - } - cRS.getConf().setKeyCols(ExprNodeDescUtils.backtrack(cKeysInParentRS, cRS, pRS)); - - // Backtrack partition columns of cRS to pRS - // If we cannot backtrack any of the columns, bail out - List cPartitionInParentRS = ExprNodeDescUtils.backtrack( - cConf.getPartitionCols(), cRS, pRS); - for (int i = 0; i < cPartitionInParentRS.size(); i++) { - ExprNodeDesc pexpr = cPartitionInParentRS.get(i); - if (pexpr == null) { - // We cannot backtrack the expression, we bail out - return false; - } - } - cRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(cPartitionInParentRS, cRS, pRS)); - - // Backtrack value columns of cRS to pRS - // If we cannot backtrack any of the columns, bail out - List cValueInParentRS = ExprNodeDescUtils.backtrack( - cConf.getValueCols(), cRS, pRS); - for (int i = 0; i < cValueInParentRS.size(); i++) { - ExprNodeDesc pexpr = cValueInParentRS.get(i); - if (pexpr == null) { - // We cannot backtrack the expression, we bail out - return false; - } - } - cRS.getConf().setValueCols(ExprNodeDescUtils.backtrack(cValueInParentRS, cRS, pRS)); - - // Backtrack bucket columns of cRS to pRS (if any) - // If we cannot backtrack any of the columns, bail out - if (cConf.getBucketCols() != null) { - List cBucketInParentRS = ExprNodeDescUtils.backtrack( - cConf.getBucketCols(), cRS, pRS); - for (int i = 0; i < cBucketInParentRS.size(); i++) { - ExprNodeDesc pexpr = cBucketInParentRS.get(i); - if (pexpr == null) { - // We cannot backtrack the expression, we bail out - return false; - } - } - cRS.getConf().setBucketCols(ExprNodeDescUtils.backtrack(cBucketInParentRS, cRS, pRS)); - } - - // Update column expression map - for (Entry e : cRS.getColumnExprMap().entrySet()) { - e.setValue(ExprNodeDescUtils.backtrack(e.getValue(), cRS, pRS)); - } - - // Replace pRS with cRS and remove operator sequence from pRS to cRS - // Recall that the sequence must be pRS-SEL*-cRS - parent = cRS.getParentOperators().get(0); - while (parent != pRS) { - dedupCtx.addRemovedOperator(parent); - parent = parent.getParentOperators().get(0); - } - dedupCtx.addRemovedOperator(pRS); - cRS.getParentOperators().clear(); - for (Operator op : pRS.getParentOperators()) { - op.replaceChild(pRS, cRS); - cRS.getParentOperators().add(op); - } - pRS.getParentOperators().clear(); - pRS.getChildOperators().clear(); - - return true; - } } static class GroupbyReducerProc extends AbsctractReducerReducerProc { @@ -652,7 +190,7 @@ public Object process(ReduceSinkOperator cRS, ReduceSinkDeduplicateProcCtx dedup ReduceSinkOperator pRS = CorrelationUtilities.findPossibleParent( pGBY, ReduceSinkOperator.class, dedupCtx.trustScript()); - if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) { + if (pRS != null && ReduceSinkDeDuplicationUtils.merge(cRS, pRS, dedupCtx.minReducer())) { CorrelationUtilities.replaceReduceSinkWithSelectOperator( cRS, dedupCtx.getPctx(), dedupCtx); pRS.getConf().setDeduplicated(true); @@ -675,7 +213,7 @@ public Object process(ReduceSinkOperator cRS, GroupByOperator cGBY, } ReduceSinkOperator pRS = CorrelationUtilities.getSingleParent(pGBY, ReduceSinkOperator.class); - if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) { + if (pRS != null && ReduceSinkDeDuplicationUtils.merge(cRS, pRS, dedupCtx.minReducer())) { CorrelationUtilities.removeReduceSinkForGroupBy( cRS, cGBY, dedupCtx.getPctx(), dedupCtx); pRS.getConf().setDeduplicated(true); @@ -693,7 +231,7 @@ public Object process(ReduceSinkOperator cRS, ReduceSinkDeduplicateProcCtx dedup throws SemanticException { JoinOperator pJoin = CorrelationUtilities.findPossibleParent(cRS, JoinOperator.class, dedupCtx.trustScript()); - if (pJoin != null && merge(cRS, pJoin, dedupCtx.minReducer())) { + if (pJoin != null && ReduceSinkDeDuplicationUtils.merge(cRS, pJoin, dedupCtx.minReducer())) { pJoin.getConf().setFixedAsSorted(true); CorrelationUtilities.replaceReduceSinkWithSelectOperator( cRS, dedupCtx.getPctx(), dedupCtx); @@ -717,7 +255,7 @@ public Object process(ReduceSinkOperator cRS, GroupByOperator cGBY, JoinOperator pJoin = CorrelationUtilities.findPossibleParent( start, JoinOperator.class, dedupCtx.trustScript()); - if (pJoin != null && merge(cRS, pJoin, dedupCtx.minReducer())) { + if (pJoin != null && ReduceSinkDeDuplicationUtils.merge(cRS, pJoin, dedupCtx.minReducer())) { pJoin.getConf().setFixedAsSorted(true); CorrelationUtilities.removeReduceSinkForGroupBy( cRS, cGBY, dedupCtx.getPctx(), dedupCtx); @@ -744,11 +282,11 @@ public Object process(ReduceSinkOperator cRS, ReduceSinkDeduplicateProcCtx dedup cRS, ReduceSinkOperator.class, dedupCtx.trustScript()); if (pRS != null) { // Try extended deduplication - if (aggressiveDedup(cRS, pRS, dedupCtx)) { + if (ReduceSinkDeDuplicationUtils.aggressiveDedup(cRS, pRS, dedupCtx)) { return true; } // Normal deduplication - if (merge(cRS, pRS, dedupCtx.minReducer())) { + if (ReduceSinkDeDuplicationUtils.merge(cRS, pRS, dedupCtx.minReducer())) { CorrelationUtilities.replaceReduceSinkWithSelectOperator( cRS, dedupCtx.getPctx(), dedupCtx); pRS.getConf().setDeduplicated(true); @@ -767,7 +305,7 @@ public Object process(ReduceSinkOperator cRS, GroupByOperator cGBY, ReduceSinkOperator pRS = CorrelationUtilities.findPossibleParent( start, ReduceSinkOperator.class, dedupCtx.trustScript()); - if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) { + if (pRS != null && ReduceSinkDeDuplicationUtils.merge(cRS, pRS, dedupCtx.minReducer())) { if (dedupCtx.getPctx().getConf().getBoolVar(HiveConf.ConfVars.HIVEGROUPBYSKEW)) { return false; } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplicationUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplicationUtils.java new file mode 100644 index 0000000..8f55369 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplicationUtils.java @@ -0,0 +1,584 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.correlation; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.PTFOperator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.SelectOperator; +import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication.ReduceSinkDeduplicateProcCtx; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; +import org.apache.hadoop.hive.ql.plan.TableDesc; + +import com.google.common.collect.ImmutableList; + +public class ReduceSinkDeDuplicationUtils { + + // for JOIN-RS case, it's not possible generally to merge if child has + // less key/partition columns than parents + public static boolean merge(ReduceSinkOperator cRS, JoinOperator pJoin, int minReducer) + throws SemanticException { + List> parents = pJoin.getParentOperators(); + ReduceSinkOperator[] pRSs = parents.toArray(new ReduceSinkOperator[parents.size()]); + ReduceSinkDesc cRSc = cRS.getConf(); + for (ReduceSinkOperator pRSNs : pRSs) { + ReduceSinkDesc pRSNc = pRSNs.getConf(); + if (cRSc.getKeyCols().size() != pRSNc.getKeyCols().size()) { + return false; + } + if (cRSc.getPartitionCols().size() != pRSNc.getPartitionCols().size()) { + return false; + } + Integer moveReducerNumTo = checkNumReducer(cRSc.getNumReducers(), pRSNc.getNumReducers()); + if (moveReducerNumTo == null || + moveReducerNumTo > 0 && cRSc.getNumReducers() < minReducer) { + return false; + } + + Integer moveRSOrderTo = checkOrder(true, cRSc.getOrder(), pRSNc.getOrder(), + cRSc.getNullOrder(), pRSNc.getNullOrder()); + if (moveRSOrderTo == null) { + return false; + } + } + + boolean[] sorted = CorrelationUtilities.getSortedTags(pJoin); + + int cKeySize = cRSc.getKeyCols().size(); + for (int i = 0; i < cKeySize; i++) { + ExprNodeDesc cexpr = cRSc.getKeyCols().get(i); + ExprNodeDesc[] pexprs = new ExprNodeDesc[pRSs.length]; + for (int tag = 0; tag < pRSs.length; tag++) { + pexprs[tag] = pRSs[tag].getConf().getKeyCols().get(i); + } + int found = CorrelationUtilities.indexOf(cexpr, pexprs, cRS, pRSs, sorted); + if (found != i) { + return false; + } + } + int cPartSize = cRSc.getPartitionCols().size(); + for (int i = 0; i < cPartSize; i++) { + ExprNodeDesc cexpr = cRSc.getPartitionCols().get(i); + ExprNodeDesc[] pexprs = new ExprNodeDesc[pRSs.length]; + for (int tag = 0; tag < pRSs.length; tag++) { + pexprs[tag] = pRSs[tag].getConf().getPartitionCols().get(i); + } + int found = CorrelationUtilities.indexOf(cexpr, pexprs, cRS, pRSs, sorted); + if (found != i) { + return false; + } + } + + for (ReduceSinkOperator pRS : pRSs) { + pRS.getConf().setNumReducers(cRS.getConf().getNumReducers()); + } + + return true; + } + + /** + * Current RSDedup remove/replace child RS. For key columns, + * sorting order, and the number of reducers, copy + * more specific part of configurations of child RS to that of parent RS. + * For partitioning columns, if both child RS and parent RS have been assigned + * partitioning columns, we will choose the more general partitioning columns. + * If parent RS has not been assigned any partitioning column, we will use + * partitioning columns (if exist) of child RS. + */ + public static boolean merge(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer) + throws SemanticException { + int[] result = extractMergeDirections(cRS, pRS, minReducer); + if (result == null) { + return false; + } + + if (result[0] > 0) { + // The sorting columns of the child RS are more specific than + // those of the parent RS. Assign sorting columns of the child RS + // to the parent RS. + List childKCs = cRS.getConf().getKeyCols(); + pRS.getConf().setKeyCols(ExprNodeDescUtils.backtrack(childKCs, cRS, pRS)); + } + + if (result[1] < 0) { + // The partitioning columns of the parent RS are more specific than + // those of the child RS. + List childPCs = cRS.getConf().getPartitionCols(); + if (childPCs != null && !childPCs.isEmpty()) { + // If partitioning columns of the child RS are assigned, + // assign these to the partitioning columns of the parent RS. + pRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(childPCs, cRS, pRS)); + } + } else if (result[1] > 0) { + // The partitioning columns of the child RS are more specific than + // those of the parent RS. + List parentPCs = pRS.getConf().getPartitionCols(); + if (parentPCs == null || parentPCs.isEmpty()) { + // If partitioning columns of the parent RS are not assigned, + // assign partitioning columns of the child RS to the parent RS. + ArrayList childPCs = cRS.getConf().getPartitionCols(); + pRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(childPCs, cRS, pRS)); + } + } + + if (result[2] > 0) { + // The sorting order of the child RS is more specific than + // that of the parent RS. Assign the sorting order of the child RS + // to the parent RS. + if (result[0] <= 0) { + // Sorting columns of the parent RS are more specific than those of the + // child RS but Sorting order of the child RS is more specific than + // that of the parent RS. + throw new SemanticException("Sorting columns and order don't match. " + + "Try set " + HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION + "=false;"); + } + pRS.getConf().setOrder(cRS.getConf().getOrder()); + pRS.getConf().setNullOrder(cRS.getConf().getNullOrder()); + } else { + // The sorting order of the parent RS is more specific or they are equal. + // We will copy the order from the child RS, and then fill in the order + // of the rest of columns with the one taken from parent RS. + StringBuilder order = new StringBuilder(cRS.getConf().getOrder()); + StringBuilder orderNull = new StringBuilder(cRS.getConf().getNullOrder()); + order.append(pRS.getConf().getOrder().substring(order.length())); + orderNull.append(pRS.getConf().getNullOrder().substring(orderNull.length())); + pRS.getConf().setOrder(order.toString()); + pRS.getConf().setNullOrder(orderNull.toString()); + } + + if (result[3] > 0) { + // The number of reducers of the child RS is more specific than + // that of the parent RS. Assign the number of reducers of the child RS + // to the parent RS. + pRS.getConf().setNumReducers(cRS.getConf().getNumReducers()); + } + + if (result[4] > 0) { + // This case happens only when pRS key is empty in which case we can use + // number of distribution keys and key serialization info from cRS + if (pRS.getConf().getKeyCols() != null && pRS.getConf().getKeyCols().size() == 0 + && cRS.getConf().getKeyCols() != null && cRS.getConf().getKeyCols().size() == 0) { + // As setNumDistributionKeys is a subset of keycols, the size should + // be 0 too. This condition maybe too strict. We may extend it in the + // future. + TableDesc keyTable = PlanUtils.getReduceKeyTableDesc(new ArrayList(), pRS + .getConf().getOrder(), pRS.getConf().getNullOrder()); + pRS.getConf().setKeySerializeInfo(keyTable); + } + } + return true; + } + + /** + * This is a more strict version of the merge check, where: + * - cRS and pRS should have exactly the same keys in the same positions, and + * - cRS and pRS should have exactly the same partition columns in the same positions, and + * - cRS and pRS should have exactly the same bucket columns in the same positions, and + * - cRS and pRS should sort in the same direction + */ + public static boolean strictMerge(ReduceSinkOperator cRS, ReduceSinkOperator pRS) + throws SemanticException { + return strictMerge(cRS, ImmutableList.of(pRS)); + } + + public static boolean strictMerge(ReduceSinkOperator cRS, List pRSs) + throws SemanticException { + ReduceSinkDesc cRSc = cRS.getConf(); + for (ReduceSinkOperator pRS : pRSs) { + ReduceSinkDesc pRSc = pRS.getConf(); + if (cRSc.getKeyCols().size() != pRSc.getKeyCols().size()) { + return false; + } + if (cRSc.getPartitionCols().size() != pRSc.getPartitionCols().size()) { + return false; + } + + Integer moveRSOrderTo = checkOrder(true, cRSc.getOrder(), pRSc.getOrder(), + cRSc.getNullOrder(), pRSc.getNullOrder()); + if (moveRSOrderTo == null) { + return false; + } + + int cKeySize = cRSc.getKeyCols().size(); + for (int i = 0; i < cKeySize; i++) { + ExprNodeDesc cExpr = cRSc.getKeyCols().get(i); + ExprNodeDesc pExpr = pRSc.getKeyCols().get(i); + if (cExpr instanceof ExprNodeConstantDesc || pExpr instanceof ExprNodeConstantDesc) { + // If ckeys or pkeys have constant node expressions avoid the merge. + return false; + } + ExprNodeDesc backtrackCExpr = ExprNodeDescUtils.backtrack(cExpr, cRS, pRS); + if (backtrackCExpr == null || !pExpr.isSame(backtrackCExpr)) { + return false; + } + } + + int cPartSize = cRSc.getPartitionCols().size(); + for (int i = 0; i < cPartSize; i++) { + ExprNodeDesc cExpr = cRSc.getPartitionCols().get(i); + ExprNodeDesc pExpr = pRSc.getPartitionCols().get(i); + if (cExpr instanceof ExprNodeConstantDesc || pExpr instanceof ExprNodeConstantDesc) { + // If cpartcols or ppartcols have constant node expressions avoid the merge. + return false; + } + ExprNodeDesc backtrackCExpr = ExprNodeDescUtils.backtrack(cExpr, cRS, pRS); + if (backtrackCExpr == null || !pExpr.isSame(backtrackCExpr)) { + return false; + } + } + + if (cRSc.getBucketCols() != null || pRSc.getBucketCols() != null) { + if (cRSc.getBucketCols() == null || pRSc.getBucketCols() == null) { + return false; + } + if (cRSc.getBucketCols().size() != pRSc.getBucketCols().size()) { + return false; + } + int cBucketColsSize = cRSc.getBucketCols().size(); + for (int i = 0; i < cBucketColsSize; i++) { + ExprNodeDesc cExpr = cRSc.getBucketCols().get(i); + ExprNodeDesc pExpr = pRSc.getBucketCols().get(i); + if (cExpr instanceof ExprNodeConstantDesc || pExpr instanceof ExprNodeConstantDesc) { + // If cbucketcols or pbucketcols have constant node expressions avoid the merge. + return false; + } + ExprNodeDesc backtrackCExpr = ExprNodeDescUtils.backtrack(cExpr, cRS, pRS); + if (backtrackCExpr == null || !pExpr.isSame(backtrackCExpr)) { + return false; + } + } + } + + // Meets all requirements + return true; + } + + // Default + return false; + } + + /** + * Returns merge directions between two RSs for criterias (ordering, number of reducers, + * reducer keys, partition keys). Returns null if any of categories is not mergeable. + * + * Values for each index can be -1, 0, 1 + * 1. 0 means two configuration in the category is the same + * 2. for -1, configuration of parent RS is more specific than child RS + * 3. for 1, configuration of child RS is more specific than parent RS + */ + private static int[] extractMergeDirections(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer) + throws SemanticException { + ReduceSinkDesc cConf = cRS.getConf(); + ReduceSinkDesc pConf = pRS.getConf(); + // If there is a PTF between cRS and pRS we cannot ignore the order direction + final boolean checkStrictEquality = isStrictEqualityNeeded(cRS, pRS); + Integer moveRSOrderTo = checkOrder(checkStrictEquality, cConf.getOrder(), pConf.getOrder(), + cConf.getNullOrder(), pConf.getNullOrder()); + if (moveRSOrderTo == null) { + return null; + } + // if cRS is being used for distinct - the two reduce sinks are incompatible + if (cConf.getDistinctColumnIndices().size() >= 2) { + return null; + } + Integer moveReducerNumTo = checkNumReducer(cConf.getNumReducers(), pConf.getNumReducers()); + if (moveReducerNumTo == null || + moveReducerNumTo > 0 && cConf.getNumReducers() < minReducer) { + return null; + } + List ckeys = cConf.getKeyCols(); + List pkeys = pConf.getKeyCols(); + Integer moveKeyColTo = checkExprs(ckeys, pkeys, cRS, pRS); + if (moveKeyColTo == null) { + return null; + } + List cpars = cConf.getPartitionCols(); + List ppars = pConf.getPartitionCols(); + Integer movePartitionColTo = checkExprs(cpars, ppars, cRS, pRS); + if (movePartitionColTo == null) { + return null; + } + Integer moveNumDistKeyTo = checkNumDistributionKey(cConf.getNumDistributionKeys(), + pConf.getNumDistributionKeys()); + return new int[] {moveKeyColTo, movePartitionColTo, moveRSOrderTo, + moveReducerNumTo, moveNumDistKeyTo}; + } + + private static boolean isStrictEqualityNeeded(ReduceSinkOperator cRS, ReduceSinkOperator pRS) { + Operator parent = cRS.getParentOperators().get(0); + while (parent != pRS) { + assert parent.getNumParent() == 1; + if (parent instanceof PTFOperator) { + return true; + } + parent = parent.getParentOperators().get(0); + } + return false; + } + + private static Integer checkNumDistributionKey(int cnd, int pnd) { + // number of distribution keys of cRS is chosen only when numDistKeys of pRS + // is 0 or less. In all other cases, distribution of the keys is based on + // the pRS which is more generic than cRS. + // Examples: + // case 1: if pRS sort key is (a, b) and cRS sort key is (a, b, c) and number of + // distribution keys are 2 and 3 resp. then after merge the sort keys will + // be (a, b, c) while the number of distribution keys will be 2. + // case 2: if pRS sort key is empty and number of distribution keys is 0 + // and if cRS sort key is (a, b) and number of distribution keys is 2 then + // after merge new sort key will be (a, b) and number of distribution keys + // will be 2. + if (pnd <= 0) { + return 1; + } + return 0; + } + + /** + * Overlapping part of keys should be the same between parent and child. + * And if child has more keys than parent, non-overlapping part of keys + * should be backtrackable to parent. + */ + private static Integer checkExprs(List ckeys, List pkeys, + ReduceSinkOperator cRS, ReduceSinkOperator pRS) throws SemanticException { + // If ckeys or pkeys have constant node expressions avoid the merge. + for (ExprNodeDesc ck : ckeys) { + if (ck instanceof ExprNodeConstantDesc) { + return null; + } + } + for (ExprNodeDesc pk : pkeys) { + if (pk instanceof ExprNodeConstantDesc) { + return null; + } + } + + Integer moveKeyColTo = 0; + if (ckeys == null || ckeys.isEmpty()) { + if (pkeys != null && !pkeys.isEmpty()) { + moveKeyColTo = -1; + } + } else { + if (pkeys == null || pkeys.isEmpty()) { + for (ExprNodeDesc ckey : ckeys) { + if (ExprNodeDescUtils.backtrack(ckey, cRS, pRS) == null) { + // cKey is not present in parent + return null; + } + } + moveKeyColTo = 1; + } else { + moveKeyColTo = sameKeys(ckeys, pkeys, cRS, pRS); + } + } + return moveKeyColTo; + } + + // backtrack key exprs of child to parent and compare it with parent's + protected static Integer sameKeys(List cexprs, List pexprs, + Operator child, Operator parent) throws SemanticException { + int common = Math.min(cexprs.size(), pexprs.size()); + int limit = Math.max(cexprs.size(), pexprs.size()); + int i = 0; + for (; i < common; i++) { + ExprNodeDesc pexpr = pexprs.get(i); + ExprNodeDesc cexpr = ExprNodeDescUtils.backtrack(cexprs.get(i), child, parent); + if (cexpr == null || !pexpr.isSame(cexpr)) { + return null; + } + } + for (; i < limit; i++) { + if (cexprs.size() > pexprs.size()) { + if (ExprNodeDescUtils.backtrack(cexprs.get(i), child, parent) == null) { + // cKey is not present in parent + return null; + } + } + } + return Integer.valueOf(cexprs.size()).compareTo(pexprs.size()); + } + + protected static Integer checkOrder(boolean checkStrictEquality, String corder, String porder, + String cNullOrder, String pNullOrder) { + assert corder.length() == cNullOrder.length(); + assert porder.length() == pNullOrder.length(); + if (corder == null || corder.trim().equals("")) { + if (porder == null || porder.trim().equals("")) { + return 0; + } + return -1; + } + if (porder == null || porder.trim().equals("")) { + return 1; + } + corder = corder.trim(); + porder = porder.trim(); + if (checkStrictEquality) { + // order of overlapping keys should be exactly the same + cNullOrder = cNullOrder.trim(); + pNullOrder = pNullOrder.trim(); + int target = Math.min(corder.length(), porder.length()); + if (!corder.substring(0, target).equals(porder.substring(0, target)) || + !cNullOrder.substring(0, target).equals(pNullOrder.substring(0, target))) { + return null; + } + } + return Integer.valueOf(corder.length()).compareTo(porder.length()); + } + + /** + * If number of reducers for RS is -1, the RS can have any number of reducers. + * It's generally true except for order-by or forced bucketing cases. + * if both of num-reducers are not -1, those number should be the same. + */ + protected static Integer checkNumReducer(int creduce, int preduce) { + if (creduce < 0) { + if (preduce < 0) { + return 0; + } + return -1; + } + if (preduce < 0) { + return 1; + } + if (creduce != preduce) { + return null; + } + return 0; + } + + protected static boolean aggressiveDedup(ReduceSinkOperator cRS, ReduceSinkOperator pRS, + ReduceSinkDeduplicateProcCtx dedupCtx) throws SemanticException { + assert cRS.getNumParent() == 1; + + ReduceSinkDesc cConf = cRS.getConf(); + ReduceSinkDesc pConf = pRS.getConf(); + List cKeys = cConf.getKeyCols(); + List pKeys = pConf.getKeyCols(); + + // Check that in the path between cRS and pRS, there are only Select operators + // i.e. the sequence must be pRS-SEL*-cRS + Operator parent = cRS.getParentOperators().get(0); + while (parent != pRS) { + assert parent.getNumParent() == 1; + if (!(parent instanceof SelectOperator)) { + return false; + } + parent = parent.getParentOperators().get(0); + } + + // If child keys are null or empty, we bail out + if (cKeys == null || cKeys.isEmpty()) { + return false; + } + // If parent keys are null or empty, we bail out + if (pKeys == null || pKeys.isEmpty()) { + return false; + } + + // Backtrack key columns of cRS to pRS + // If we cannot backtrack any of the columns, bail out + List cKeysInParentRS = ExprNodeDescUtils.backtrack(cKeys, cRS, pRS); + for (int i = 0; i < cKeysInParentRS.size(); i++) { + ExprNodeDesc pexpr = cKeysInParentRS.get(i); + if (pexpr == null) { + // We cannot backtrack the expression, we bail out + return false; + } + } + cRS.getConf().setKeyCols(ExprNodeDescUtils.backtrack(cKeysInParentRS, cRS, pRS)); + + // Backtrack partition columns of cRS to pRS + // If we cannot backtrack any of the columns, bail out + List cPartitionInParentRS = ExprNodeDescUtils.backtrack( + cConf.getPartitionCols(), cRS, pRS); + for (int i = 0; i < cPartitionInParentRS.size(); i++) { + ExprNodeDesc pexpr = cPartitionInParentRS.get(i); + if (pexpr == null) { + // We cannot backtrack the expression, we bail out + return false; + } + } + cRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(cPartitionInParentRS, cRS, pRS)); + + // Backtrack value columns of cRS to pRS + // If we cannot backtrack any of the columns, bail out + List cValueInParentRS = ExprNodeDescUtils.backtrack( + cConf.getValueCols(), cRS, pRS); + for (int i = 0; i < cValueInParentRS.size(); i++) { + ExprNodeDesc pexpr = cValueInParentRS.get(i); + if (pexpr == null) { + // We cannot backtrack the expression, we bail out + return false; + } + } + cRS.getConf().setValueCols(ExprNodeDescUtils.backtrack(cValueInParentRS, cRS, pRS)); + + // Backtrack bucket columns of cRS to pRS (if any) + // If we cannot backtrack any of the columns, bail out + if (cConf.getBucketCols() != null) { + List cBucketInParentRS = ExprNodeDescUtils.backtrack( + cConf.getBucketCols(), cRS, pRS); + for (int i = 0; i < cBucketInParentRS.size(); i++) { + ExprNodeDesc pexpr = cBucketInParentRS.get(i); + if (pexpr == null) { + // We cannot backtrack the expression, we bail out + return false; + } + } + cRS.getConf().setBucketCols(ExprNodeDescUtils.backtrack(cBucketInParentRS, cRS, pRS)); + } + + // Update column expression map + for (Entry e : cRS.getColumnExprMap().entrySet()) { + e.setValue(ExprNodeDescUtils.backtrack(e.getValue(), cRS, pRS)); + } + + // Replace pRS with cRS and remove operator sequence from pRS to cRS + // Recall that the sequence must be pRS-SEL*-cRS + parent = cRS.getParentOperators().get(0); + while (parent != pRS) { + dedupCtx.addRemovedOperator(parent); + parent = parent.getParentOperators().get(0); + } + dedupCtx.addRemovedOperator(pRS); + cRS.getParentOperators().clear(); + for (Operator op : pRS.getParentOperators()) { + op.replaceChild(pRS, cRS); + cRS.getParentOperators().add(op); + } + pRS.getParentOperators().clear(); + pRS.getChildOperators().clear(); + + return true; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkJoinDeDuplication.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkJoinDeDuplication.java new file mode 100644 index 0000000..d27320b --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkJoinDeDuplication.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer.correlation; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Stack; + +import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator; +import org.apache.hadoop.hive.ql.exec.MapJoinOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; +import org.apache.hadoop.hive.ql.lib.Dispatcher; +import org.apache.hadoop.hive.ql.lib.ForwardWalker; +import org.apache.hadoop.hive.ql.lib.GraphWalker; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.lib.Rule; +import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.optimizer.Transform; +import org.apache.hadoop.hive.ql.parse.ParseContext; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableList; + +/** + * Optimization to check whether any ReduceSink operator in the plan can be + * simplified so data is not shuffled/sorted if it is already shuffled/sorted. + * + * This optimization is executed after join algorithm selection logic has run, + * and it is intended to optimize new cases that cannot be optimized when + * {@link ReduceSinkDeDuplication} runs because some physical algorithms have + * not been selected. Instead of removing ReduceSink operators from the plan, + * they will be tagged, and then the execution plan compiler might take action, + * e.g., on Tez, ReduceSink operators that just need to forward data will be + * translated into a ONE-TO-ONE edge. The parallelism degree of these ReduceSink + * operators might be adjusted, as a ReduceSink operator that just forwards data + * cannot alter the degree of parallelism of the previous task. + */ +public class ReduceSinkJoinDeDuplication extends Transform { + + protected static final Logger LOG = LoggerFactory.getLogger(ReduceSinkJoinDeDuplication.class); + + protected ParseContext pGraphContext; + + @Override + public ParseContext transform(ParseContext pctx) throws SemanticException { + pGraphContext = pctx; + + ReduceSinkJoinDeDuplicateProcCtx cppCtx = new ReduceSinkJoinDeDuplicateProcCtx(pGraphContext); + + Map opRules = new LinkedHashMap(); + opRules.put(new RuleRegExp("R1", ReduceSinkOperator.getOperatorName() + "%"), + ReduceSinkJoinDeDuplicateProcFactory.getReducerMapJoinProc()); + + Dispatcher disp = new DefaultRuleDispatcher( + ReduceSinkJoinDeDuplicateProcFactory.getDefaultProc(), opRules, cppCtx); + GraphWalker ogw = new ForwardWalker(disp); + + // Create a list of topop nodes + ArrayList topNodes = new ArrayList(); + topNodes.addAll(pGraphContext.getTopOps().values()); + ogw.startWalking(topNodes, null); + return pGraphContext; + } + + protected class ReduceSinkJoinDeDuplicateProcCtx extends AbstractCorrelationProcCtx { + + public ReduceSinkJoinDeDuplicateProcCtx(ParseContext pctx) { + super(pctx); + } + } + + static class ReduceSinkJoinDeDuplicateProcFactory { + + public static NodeProcessor getReducerMapJoinProc() { + return new ReducerProc(); + } + + public static NodeProcessor getDefaultProc() { + return new DefaultProc(); + } + } + + /* + * do nothing. + */ + static class DefaultProc implements NodeProcessor { + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + return null; + } + } + + static class ReducerProc implements NodeProcessor { + + @Override + public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, + Object... nodeOutputs) throws SemanticException { + ReduceSinkJoinDeDuplicateProcCtx dedupCtx = (ReduceSinkJoinDeDuplicateProcCtx) procCtx; + ReduceSinkOperator cRS = (ReduceSinkOperator) nd; + if (cRS.getConf().isForwarding()) { + // Already set + return false; + } + if (cRS.getConf().getKeyCols().isEmpty()) { + // Not supported + return false; + } + boolean onlyPartitioning = false; + Operator cRSChild = cRS.getChildOperators().get(0); + if (cRSChild instanceof MapJoinOperator || + cRSChild instanceof CommonMergeJoinOperator) { + // If it is a MapJoin or MergeJoin, we make sure that they are on + // the reduce side, otherwise we bail out + for (Operator parent: cRSChild.getParentOperators()) { + if (!(parent instanceof ReduceSinkOperator)) { + // MapJoin and SMBJoin not supported + return false; + } + } + if (cRSChild instanceof MapJoinOperator) { + onlyPartitioning = true; + } + } + + int maxNumReducers = cRS.getConf().getNumReducers(); + ReduceSinkOperator pRS; + if (onlyPartitioning) { + pRS = CorrelationUtilities.findFirstPossibleParent( + cRS, ReduceSinkOperator.class, dedupCtx.trustScript()); + } else { + pRS = CorrelationUtilities.findFirstPossibleParentPreserveSortOrder( + cRS, ReduceSinkOperator.class, dedupCtx.trustScript()); + } + if (pRS != null) { + Operator pRSChild = pRS.getChildOperators().get(0); + if (pRSChild instanceof MapJoinOperator) { + // Handle MapJoin specially and check for all its children + MapJoinOperator pRSChildMJ = (MapJoinOperator) pRSChild; + // In this case, both should be DHJ operators as pRSChildMJ can only guarantee + // partitioned input, not sorted. + if (!pRSChildMJ.getConf().isDynamicPartitionHashJoin() || + !(cRSChild instanceof MapJoinOperator) || + !((MapJoinOperator) cRSChild).getConf().isDynamicPartitionHashJoin()) { + return false; + } + ImmutableList.Builder l = ImmutableList.builder(); + for (Operator parent: pRSChild.getParentOperators()) { + ReduceSinkOperator rsOp = (ReduceSinkOperator) parent; + l.add(rsOp); + if (rsOp.getConf().getNumReducers() > maxNumReducers) { + maxNumReducers = rsOp.getConf().getNumReducers(); + } + } + if (ReduceSinkDeDuplicationUtils.strictMerge(cRS, l.build())) { + LOG.debug("Set {} to forward data", cRS); + cRS.getConf().setForwarding(true); + propagateMaxNumReducers(dedupCtx, cRS, maxNumReducers); + return true; + } + } else if (pRS.getChildOperators().get(0) instanceof CommonMergeJoinOperator) { + // Handle MergeJoin specially and check for all its children + ImmutableList.Builder l = ImmutableList.builder(); + for (Operator parent: pRSChild.getParentOperators()) { + if (!(parent instanceof ReduceSinkOperator)) { + // SMBJoin not supported + return false; + } + ReduceSinkOperator rsOp = (ReduceSinkOperator) parent; + l.add(rsOp); + if (rsOp.getConf().getNumReducers() > maxNumReducers) { + maxNumReducers = rsOp.getConf().getNumReducers(); + } + } + if (ReduceSinkDeDuplicationUtils.strictMerge(cRS, l.build())) { + LOG.debug("Set {} to forward data", cRS); + cRS.getConf().setForwarding(true); + propagateMaxNumReducers(dedupCtx, cRS, maxNumReducers); + return true; + } + } else { + // Rest of cases + if (pRS.getConf().getNumReducers() > maxNumReducers) { + maxNumReducers = pRS.getConf().getNumReducers(); + } + if (ReduceSinkDeDuplicationUtils.strictMerge(cRS, pRS)) { + LOG.debug("Set {} to forward data", cRS); + cRS.getConf().setForwarding(true); + propagateMaxNumReducers(dedupCtx, cRS, maxNumReducers); + return true; + } + } + } + return false; + } + + private static void propagateMaxNumReducers(ReduceSinkJoinDeDuplicateProcCtx dedupCtx, + ReduceSinkOperator rsOp, int maxNumReducers) throws SemanticException { + if (rsOp == null) { + // Bail out + return; + } + if (rsOp.getChildOperators().get(0) instanceof MapJoinOperator || + rsOp.getChildOperators().get(0) instanceof CommonMergeJoinOperator) { + for (Operator p : rsOp.getChildOperators().get(0).getParentOperators()) { + ReduceSinkOperator pRSOp = (ReduceSinkOperator) p; + pRSOp.getConf().setReducerTraits(EnumSet.of(ReducerTraits.FIXED)); + pRSOp.getConf().setNumReducers(maxNumReducers); + LOG.debug("Set {} to FIXED parallelism: {}", pRSOp, maxNumReducers); + if (pRSOp.getConf().isForwarding()) { + ReduceSinkOperator newRSOp = + CorrelationUtilities.findFirstPossibleParent( + pRSOp, ReduceSinkOperator.class, dedupCtx.trustScript()); + propagateMaxNumReducers(dedupCtx, newRSOp, maxNumReducers); + } + } + } else { + rsOp.getConf().setReducerTraits(EnumSet.of(ReducerTraits.FIXED)); + rsOp.getConf().setNumReducers(maxNumReducers); + LOG.debug("Set {} to FIXED parallelism: {}", rsOp, maxNumReducers); + if (rsOp.getConf().isForwarding()) { + ReduceSinkOperator newRSOp = + CorrelationUtilities.findFirstPossibleParent( + rsOp, ReduceSinkOperator.class, dedupCtx.trustScript()); + propagateMaxNumReducers(dedupCtx, newRSOp, maxNumReducers); + } + } + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index 6423a6d..1b0a2f0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -502,6 +502,9 @@ public static void removeBranch(Operator event) { } public static EdgeType determineEdgeType(BaseWork preceedingWork, BaseWork followingWork, ReduceSinkOperator reduceSinkOperator) { + if(reduceSinkOperator.getConf().isForwarding()) { + return EdgeType.ONE_TO_ONE_EDGE; + } if (followingWork instanceof ReduceWork) { // Ideally there should be a better way to determine that the followingWork contains // a dynamic partitioned hash join, but in some cases (createReduceWork()) it looks like diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 888627a..5614c26 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -77,6 +77,7 @@ import org.apache.hadoop.hive.ql.optimizer.RemoveDynamicPruningBySize; import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism; import org.apache.hadoop.hive.ql.optimizer.SharedWorkOptimizer; +import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkJoinDeDuplication; import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits; import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer; import org.apache.hadoop.hive.ql.optimizer.physical.CrossProductCheck; @@ -154,6 +155,16 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, runStatsDependentOptimizations(procCtx, inputs, outputs); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run the optimizations that use stats for optimization"); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + if(procCtx.conf.getBoolVar(ConfVars.HIVEOPTJOINREDUCEDEDUPLICATION)) { + new ReduceSinkJoinDeDuplication().transform(procCtx.parseContext); + } + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run reduce sink after join algorithm selection"); + + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + runRemoveDynamicPruningOptimization(procCtx, inputs, outputs); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run remove dynamic pruning by size"); + // Removing semijoin optimization when it may not be beneficial removeSemijoinOptimizationByBenefit(procCtx); @@ -414,6 +425,25 @@ private void runStatsDependentOptimizations(OptimizeTezProcContext procCtx, opRules.put(new RuleRegExp("Convert Join to Map-join", JoinOperator.getOperatorName() + "%"), new ConvertJoinMapJoin()); + // The dispatcher fires the processor corresponding to the closest matching + // rule and passes the context along + Dispatcher disp = new DefaultRuleDispatcher(null, opRules, procCtx); + List topNodes = new ArrayList(); + topNodes.addAll(procCtx.parseContext.getTopOps().values()); + GraphWalker ogw = new ForwardWalker(disp); + ogw.startWalking(topNodes, null); + } + + private void runRemoveDynamicPruningOptimization(OptimizeTezProcContext procCtx, + Set inputs, Set outputs) throws SemanticException { + + // Sequence of TableScan operators to be walked + Deque> deque = new LinkedList>(); + deque.addAll(procCtx.parseContext.getTopOps().values()); + + // create a walker which walks the tree in a DFS manner while maintaining + // the operator stack. + Map opRules = new LinkedHashMap(); opRules.put( new RuleRegExp("Remove dynamic pruning by size", AppMasterEventOperator.getOperatorName() + "%"), diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java index 24636c5..d08e700 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java @@ -98,6 +98,7 @@ //flag used to control how TopN handled for PTF/Windowing partitions. private boolean isPTFReduceSink = false; private boolean skipTag; // Skip writing tags when feeding into mapjoin hashtable + private boolean forwarding; // Whether this RS can forward records directly instead of shuffling/sorting public static enum ReducerTraits { UNSET(0), // unset @@ -432,6 +433,14 @@ public boolean getSkipTag() { return skipTag; } + public void setForwarding(boolean forwarding) { + this.forwarding = forwarding; + } + + public boolean isForwarding() { + return forwarding; + } + @Explain(displayName = "auto parallelism", explainLevels = { Level.EXTENDED }) public final boolean isAutoParallel() { return (this.reduceTraits.contains(ReducerTraits.AUTOPARALLEL)); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java index a55c708..b695f0f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java @@ -28,6 +28,7 @@ CONTAINS, CUSTOM_EDGE, CUSTOM_SIMPLE_EDGE, + ONE_TO_ONE_EDGE } private HiveConf hiveConf; @@ -102,4 +103,8 @@ public boolean isSlowStart() { public void setSlowStart(boolean slowStart) { this.isSlowStart = slowStart; } + + public void setEdgeType(EdgeType type) { + this.edgeType = type; + } } diff --git ql/src/test/queries/clientpositive/partialdhj.q ql/src/test/queries/clientpositive/partialdhj.q new file mode 100644 index 0000000..c4fe1f7 --- /dev/null +++ ql/src/test/queries/clientpositive/partialdhj.q @@ -0,0 +1,54 @@ +set hive.auto.convert.join=true; +set hive.strict.checks.cartesian.product=false; +set hive.merge.nway.joins=false; +set hive.optimize.shared.work=false; +set hive.optimize.dynamic.partition.hashjoin=true; +set hive.auto.convert.join.hashtable.max.entries=10; + +-- ONE_TO_ONE ON BIG TABLE SIDE +EXPLAIN +SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value); + +SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value); + +-- ONE_TO_ONE ON SMALL TABLE SIDE +EXPLAIN +SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value); + +SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value); diff --git ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out index c563976..8bc280c 100644 --- ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out @@ -322,7 +322,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out index 799de73..ecfa860 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out @@ -22,7 +22,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -190,7 +190,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -358,7 +358,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -524,7 +524,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -690,7 +690,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -856,7 +856,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1022,7 +1022,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1192,7 +1192,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1364,7 +1364,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) @@ -1552,7 +1552,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) @@ -1738,7 +1738,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1919,7 +1919,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out index 611ba06..33aa7ac 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out @@ -22,7 +22,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) @@ -239,7 +239,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) @@ -667,7 +667,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) @@ -884,7 +884,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) diff --git ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out index 177cc97..a47a7d3 100644 --- ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out +++ ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out @@ -25,7 +25,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -269,7 +269,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -733,7 +733,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -874,7 +874,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1015,7 +1015,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1196,7 +1196,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1379,7 +1379,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1582,7 +1582,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1785,7 +1785,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2003,7 +2003,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2223,7 +2223,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 4 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2441,7 +2441,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 4 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2660,9 +2660,9 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2914,9 +2914,9 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 5 <- Map 8 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -3167,7 +3167,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE) @@ -3424,7 +3424,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/explainuser_1.q.out ql/src/test/results/clientpositive/llap/explainuser_1.q.out index eb1268c..b36fe2a 100644 --- ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -1737,7 +1737,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Stage-0 @@ -1752,8 +1752,8 @@ Stage-0 predicate:_col4 is null Merge Join Operator [MERGEJOIN_21] (rows=250 width=178) Conds:RS_13._col0, _col1=RS_14._col0, _col1(Left Outer),Output:["_col0","_col1","_col4"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] + <-Reducer 2 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_13] PartitionCols:_col0, _col1 Group By Operator [GBY_4] (rows=250 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 @@ -1766,8 +1766,8 @@ Stage-0 Output:["key","value"] TableScan [TS_0] (rows=500 width=178) default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_14] + <-Reducer 4 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_14] PartitionCols:_col0, _col1 Select Operator [SEL_12] (rows=83 width=182) Output:["_col0","_col1","_col2"] @@ -1948,7 +1948,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) +Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Stage-0 @@ -1986,8 +1986,8 @@ Stage-0 predicate:((l_linenumber = 1) and l_partkey is not null) TableScan [TS_7] (rows=100 width=16) default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_partkey","l_suppkey","l_linenumber"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_13] + <-Reducer 2 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_13] PartitionCols:_col0 Group By Operator [GBY_5] (rows=50 width=4) Output:["_col0"],keys:KEY._col0 @@ -2165,7 +2165,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 5 (SIMPLE_EDGE) @@ -2209,8 +2209,8 @@ Stage-0 predicate:(key > '2') TableScan [TS_2] (rows=500 width=87) default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_20] + <-Reducer 7 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_20] PartitionCols:_col0 Select Operator [SEL_15] (rows=83 width=91) Output:["_col0","_col1"] @@ -2244,8 +2244,8 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) +Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) @@ -2273,8 +2273,8 @@ Stage-0 Output:["_col0","_col1","_col2"] TableScan [TS_0] (rows=26 width=223) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"] - <-Reducer 4 [SIMPLE_EDGE] llap - SHUFFLE [RS_19] + <-Reducer 4 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_19] PartitionCols:_col0 Group By Operator [GBY_7] (rows=2 width=114) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 @@ -2288,8 +2288,8 @@ Stage-0 Filter Operator [FIL_29] (rows=8 width=223) predicate:((p_size < 10) and p_mfgr is not null) Please refer to the previous TableScan [TS_0] - <-Reducer 5 [SIMPLE_EDGE] llap - SHUFFLE [RS_22] + <-Reducer 5 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_22] PartitionCols:_col0, _col1 Select Operator [SEL_17] (rows=4 width=223) Output:["_col0","_col1","_col2"] @@ -2409,7 +2409,7 @@ Plan optimized by CBO. Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) -Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) +Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE) @@ -2451,8 +2451,8 @@ Stage-0 Output:["p_mfgr","p_retailprice"] TableScan [TS_0] (rows=26 width=106) default@part,b,Tbl:COMPLETE,Col:COMPLETE,Output:["p_mfgr","p_retailprice"] - <-Reducer 7 [SIMPLE_EDGE] llap - SHUFFLE [RS_28] + <-Reducer 7 [ONE_TO_ONE_EDGE] llap + FORWARD [RS_28] PartitionCols:_col0 Group By Operator [GBY_16] (rows=1 width=24) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/llap/mrr.q.out ql/src/test/results/clientpositive/llap/mrr.q.out index c422021..bfa26e4 100644 --- ql/src/test/results/clientpositive/llap/mrr.q.out +++ ql/src/test/results/clientpositive/llap/mrr.q.out @@ -1293,7 +1293,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE) diff --git ql/src/test/results/clientpositive/llap/parallel.q.out ql/src/test/results/clientpositive/llap/parallel.q.out index 7dba122..aea9417 100644 --- ql/src/test/results/clientpositive/llap/parallel.q.out +++ ql/src/test/results/clientpositive/llap/parallel.q.out @@ -38,7 +38,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/llap/parallel_colstats.q.out ql/src/test/results/clientpositive/llap/parallel_colstats.q.out index 6c95f3a..57498a6 100644 --- ql/src/test/results/clientpositive/llap/parallel_colstats.q.out +++ ql/src/test/results/clientpositive/llap/parallel_colstats.q.out @@ -40,7 +40,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/partialdhj.q.out ql/src/test/results/clientpositive/llap/partialdhj.q.out new file mode 100644 index 0000000..4e62c4f --- /dev/null +++ ql/src/test/results/clientpositive/llap/partialdhj.q.out @@ -0,0 +1,460 @@ +PREHOOK: query: EXPLAIN +SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 KEY.reducesinkkey0 (type: string) + 1 KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Map 4 + Statistics: Num rows: 32 Data size: 2848 Basic stats: COMPLETE Column stats: COMPLETE + HybridGraceHashJoin: true + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 KEY.reducesinkkey0 (type: string) + 1 KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 + input vertices: + 0 Reducer 3 + Statistics: Num rows: 25 Data size: 6675 Basic stats: COMPLETE Column stats: COMPLETE + HybridGraceHashJoin: true + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 6675 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +JOIN src +ON (a.value = src.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +val_311 311 val_311 +val_165 165 val_165 +val_409 409 val_409 +val_98 98 val_98 +val_484 484 val_484 +val_311 311 val_311 +val_311 311 val_311 +val_165 165 val_165 +val_98 98 val_98 +val_409 409 val_409 +val_409 409 val_409 +val_27 27 val_27 +val_146 146 val_146 +val_146 146 val_146 +val_238 238 val_238 +val_238 238 val_238 +val_193 193 val_193 +val_273 273 val_273 +val_193 193 val_193 +val_273 273 val_273 +val_193 193 val_193 +val_273 273 val_273 +val_406 406 val_406 +val_406 406 val_406 +val_406 406 val_406 +val_406 406 val_406 +val_66 66 val_66 +val_213 213 val_213 +val_213 213 val_213 +val_278 278 val_278 +val_401 401 val_401 +val_278 278 val_278 +val_401 401 val_401 +val_401 401 val_401 +val_401 401 val_401 +val_401 401 val_401 +val_150 150 val_150 +val_255 255 val_255 +val_265 265 val_265 +val_255 255 val_255 +val_265 265 val_265 +PREHOOK: query: EXPLAIN +SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Map 6 (CUSTOM_SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 2225 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 KEY.reducesinkkey0 (type: string) + 1 KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 + input vertices: + 1 Reducer 5 + Statistics: Num rows: 25 Data size: 6675 Basic stats: COMPLETE Column stats: COMPLETE + HybridGraceHashJoin: true + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 6675 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 KEY.reducesinkkey0 (type: string) + 1 KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + input vertices: + 1 Map 6 + Statistics: Num rows: 32 Data size: 2848 Basic stats: COMPLETE Column stats: COMPLETE + HybridGraceHashJoin: true + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 16 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * +FROM src +JOIN ( + SELECT a.value + FROM src1 a + JOIN src1 b + ON (a.value = b.value) + GROUP BY a.value +) a +ON (src.value = a.value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### +311 val_311 val_311 +165 val_165 val_165 +409 val_409 val_409 +98 val_98 val_98 +484 val_484 val_484 +311 val_311 val_311 +311 val_311 val_311 +165 val_165 val_165 +98 val_98 val_98 +409 val_409 val_409 +409 val_409 val_409 +27 val_27 val_27 +146 val_146 val_146 +146 val_146 val_146 +238 val_238 val_238 +238 val_238 val_238 +193 val_193 val_193 +273 val_273 val_273 +193 val_193 val_193 +273 val_273 val_273 +193 val_193 val_193 +273 val_273 val_273 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +406 val_406 val_406 +66 val_66 val_66 +213 val_213 val_213 +213 val_213 val_213 +278 val_278 val_278 +401 val_401 val_401 +278 val_278 val_278 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +401 val_401 val_401 +150 val_150 val_150 +255 val_255 val_255 +265 val_265 val_265 +255 val_255 val_255 +265 val_265 val_265 diff --git ql/src/test/results/clientpositive/llap/subquery_in.q.out ql/src/test/results/clientpositive/llap/subquery_in.q.out index 5c94c6e..53651c4 100644 --- ql/src/test/results/clientpositive/llap/subquery_in.q.out +++ ql/src/test/results/clientpositive/llap/subquery_in.q.out @@ -744,7 +744,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -901,7 +901,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1249,7 +1249,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1380,7 +1380,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1523,7 +1523,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1657,7 +1657,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1778,7 +1778,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2082,7 +2082,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2212,7 +2212,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2339,7 +2339,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2730,7 +2730,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) @@ -2955,7 +2955,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) @@ -3111,7 +3111,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) @@ -4115,7 +4115,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) @@ -4578,8 +4578,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Map 5 (SIMPLE_EDGE) @@ -4831,8 +4831,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE) @@ -5591,7 +5591,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 9 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) diff --git ql/src/test/results/clientpositive/llap/subquery_multi.q.out ql/src/test/results/clientpositive/llap/subquery_multi.q.out index bbc8a5b..5673f0e 100644 --- ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -88,7 +88,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -261,9 +261,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Map 5 (CUSTOM_SIMPLE_EDGE) Reducer 8 <- Map 5 (SIMPLE_EDGE) @@ -462,9 +462,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) Reducer 9 <- Map 7 (SIMPLE_EDGE) @@ -696,9 +696,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) Reducer 9 <- Map 7 (SIMPLE_EDGE) @@ -1134,7 +1134,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) @@ -1316,7 +1316,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) @@ -1472,7 +1472,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) @@ -1665,9 +1665,9 @@ STAGE PLANS: Edges: Reducer 10 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 11 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE) @@ -1981,7 +1981,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2156,9 +2156,9 @@ STAGE PLANS: Edges: Reducer 10 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) - Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 11 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE) @@ -2472,9 +2472,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE) Reducer 9 <- Map 7 (SIMPLE_EDGE) @@ -2714,7 +2714,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2901,7 +2901,7 @@ STAGE PLANS: Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE) Reducer 11 <- Map 9 (CUSTOM_SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE), Reducer 10 (CUSTOM_SIMPLE_EDGE) Reducer 8 <- Reducer 11 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) @@ -3873,7 +3873,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -4036,7 +4036,7 @@ STAGE PLANS: Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 10 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) diff --git ql/src/test/results/clientpositive/llap/subquery_notin.q.out ql/src/test/results/clientpositive/llap/subquery_notin.q.out index e4aee8e..31f8f54 100644 --- ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -25,7 +25,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -317,8 +317,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE) @@ -893,8 +893,8 @@ STAGE PLANS: Edges: Reducer 10 <- Map 5 (SIMPLE_EDGE) Reducer 11 <- Reducer 10 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (SIMPLE_EDGE) @@ -1478,7 +1478,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1640,8 +1640,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1841,7 +1841,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 4 (SIMPLE_EDGE) @@ -2056,7 +2056,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 4 (SIMPLE_EDGE) @@ -2279,7 +2279,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) @@ -2505,7 +2505,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 4 (SIMPLE_EDGE) @@ -2705,8 +2705,8 @@ STAGE PLANS: Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) Reducer 11 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) @@ -2975,8 +2975,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -3140,8 +3140,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -3311,11 +3311,11 @@ STAGE PLANS: Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) Reducer 11 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) + Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (ONE_TO_ONE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -3589,7 +3589,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) @@ -3862,8 +3862,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE) @@ -4162,7 +4162,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) @@ -4390,7 +4390,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE) @@ -4615,7 +4615,7 @@ STAGE PLANS: Map 8 <- Union 9 (CONTAINS) Reducer 10 <- Union 9 (SIMPLE_EDGE) Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 3 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 6 <- Union 5 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -5324,7 +5324,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) @@ -5581,7 +5581,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -5736,8 +5736,8 @@ STAGE PLANS: Edges: Reducer 10 <- Map 9 (SIMPLE_EDGE) Reducer 11 <- Map 9 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) @@ -6036,8 +6036,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -6251,10 +6251,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 5 <- Map 3 (SIMPLE_EDGE) - Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -6477,10 +6477,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -6667,8 +6667,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -6827,7 +6827,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -6982,7 +6982,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -7145,8 +7145,8 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 10 <- Map 8 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 9 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 6 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 10 (CUSTOM_SIMPLE_EDGE) diff --git ql/src/test/results/clientpositive/llap/subquery_scalar.q.out ql/src/test/results/clientpositive/llap/subquery_scalar.q.out index 1556114..06dec8e 100644 --- ql/src/test/results/clientpositive/llap/subquery_scalar.q.out +++ ql/src/test/results/clientpositive/llap/subquery_scalar.q.out @@ -1317,7 +1317,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1447,7 +1447,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -1574,7 +1574,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1691,7 +1691,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) @@ -1885,7 +1885,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) @@ -2083,7 +2083,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2263,7 +2263,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -3155,7 +3155,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -3336,7 +3336,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) Reducer 8 <- Map 7 (CUSTOM_SIMPLE_EDGE) Reducer 9 <- Map 7 (SIMPLE_EDGE) @@ -3572,8 +3572,8 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 5 (SIMPLE_EDGE) Reducer 9 <- Map 8 (SIMPLE_EDGE) @@ -3800,8 +3800,8 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4009,8 +4009,8 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4211,7 +4211,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -4370,7 +4370,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4673,7 +4673,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4795,7 +4795,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4911,8 +4911,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -5094,7 +5094,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) @@ -5775,7 +5775,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/subquery_select.q.out ql/src/test/results/clientpositive/llap/subquery_select.q.out index eafb77a..fd10898 100644 --- ql/src/test/results/clientpositive/llap/subquery_select.q.out +++ ql/src/test/results/clientpositive/llap/subquery_select.q.out @@ -200,7 +200,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) @@ -586,8 +586,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) Reducer 6 <- Map 1 (SIMPLE_EDGE) @@ -983,7 +983,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1254,7 +1254,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1391,7 +1391,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1529,7 +1529,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -1804,7 +1804,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE) @@ -1979,7 +1979,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) - Reducer 3 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 3 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2154,7 +2154,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) Reducer 6 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2608,7 +2608,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -2743,7 +2743,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -3069,7 +3069,7 @@ STAGE PLANS: Edges: Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE) Reducer 11 <- Map 9 (CUSTOM_SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 4 <- Reducer 10 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Reducer 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) @@ -3362,7 +3362,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4812,7 +4812,7 @@ STAGE PLANS: Reducer 11 <- Map 10 (SIMPLE_EDGE) Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) - Reducer 5 <- Reducer 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 5 <- Reducer 11 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 9 <- Map 8 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -5107,10 +5107,10 @@ STAGE PLANS: Edges: Reducer 10 <- Map 7 (SIMPLE_EDGE) Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) - Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 5 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 12 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE) Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) Reducer 8 <- Map 7 (SIMPLE_EDGE) Reducer 9 <- Map 7 (SIMPLE_EDGE) @@ -5478,7 +5478,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/llap/subquery_views.q.out ql/src/test/results/clientpositive/llap/subquery_views.q.out index a2b3fd2..094d6b2 100644 --- ql/src/test/results/clientpositive/llap/subquery_views.q.out +++ ql/src/test/results/clientpositive/llap/subquery_views.q.out @@ -124,15 +124,15 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 10 <- Reducer 13 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 10 <- Reducer 13 (ONE_TO_ONE_EDGE), Reducer 9 (SIMPLE_EDGE) Reducer 12 <- Map 11 (SIMPLE_EDGE) Reducer 13 <- Map 11 (SIMPLE_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) Reducer 5 <- Map 1 (SIMPLE_EDGE) - Reducer 6 <- Reducer 12 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 6 <- Reducer 12 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE) Reducer 8 <- Map 1 (SIMPLE_EDGE) Reducer 9 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out index 6dcf53f..67cd110 100644 --- ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_group_by.q.out @@ -151,7 +151,7 @@ STAGE PLANS: Map 5 <- Union 2 (CONTAINS) Map 6 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 8 <- Map 10 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE) #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out index 4b1e92d..b4386c8 100644 --- ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out @@ -413,7 +413,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -547,7 +547,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator + className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE @@ -604,7 +604,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Reduce Sink Vectorization: - className: VectorReduceSinkLongOperator + className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/perf/query58.q.out ql/src/test/results/clientpositive/perf/query58.q.out index 5b37f36..a3e2e7e 100644 --- ql/src/test/results/clientpositive/perf/query58.q.out +++ ql/src/test/results/clientpositive/perf/query58.q.out @@ -136,9 +136,9 @@ Reducer 10 <- Reducer 9 (SIMPLE_EDGE) Reducer 11 <- Map 33 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE) -Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 22 (ONE_TO_ONE_EDGE) +Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 26 (ONE_TO_ONE_EDGE) +Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 30 (ONE_TO_ONE_EDGE) Reducer 19 <- Map 18 (CUSTOM_SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 20 <- Map 31 (CUSTOM_SIMPLE_EDGE), Reducer 19 (CUSTOM_SIMPLE_EDGE) @@ -203,8 +203,8 @@ Stage-0 predicate:(d_date is not null and d_date_sk is not null) TableScan [TS_6] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 26 [SIMPLE_EDGE] - SHUFFLE [RS_86] + <-Reducer 26 [ONE_TO_ONE_EDGE] + FORWARD [RS_86] PartitionCols:_col0 Group By Operator [GBY_83] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 @@ -314,8 +314,8 @@ Stage-0 SHUFFLE [RS_135] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_8] - <-Reducer 30 [SIMPLE_EDGE] - SHUFFLE [RS_136] + <-Reducer 30 [ONE_TO_ONE_EDGE] + FORWARD [RS_136] PartitionCols:_col0 Group By Operator [GBY_133] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 @@ -377,8 +377,8 @@ Stage-0 SHUFFLE [RS_35] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_8] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_36] + <-Reducer 22 [ONE_TO_ONE_EDGE] + FORWARD [RS_36] PartitionCols:_col0 Group By Operator [GBY_33] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query6.q.out ql/src/test/results/clientpositive/perf/query6.q.out index 6ea20e9..1c685e2 100644 --- ql/src/test/results/clientpositive/perf/query6.q.out +++ ql/src/test/results/clientpositive/perf/query6.q.out @@ -58,7 +58,7 @@ Reducer 12 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 19 (CUSTOM_SIMPLE_EDGE) Reducer 13 <- Map 20 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE) Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE) Reducer 19 <- Map 18 (SIMPLE_EDGE) -Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) +Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE) Reducer 3 <- Map 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 16 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) @@ -208,8 +208,8 @@ Stage-0 predicate:(d_date_sk is not null and d_month_seq is not null) TableScan [TS_0] (rows=73049 width=1119) default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"] - <-Reducer 9 [SIMPLE_EDGE] - SHUFFLE [RS_56] + <-Reducer 9 [ONE_TO_ONE_EDGE] + FORWARD [RS_56] PartitionCols:_col0 Group By Operator [GBY_8] (rows=9131 width=1119) Output:["_col0"],keys:KEY._col0 diff --git ql/src/test/results/clientpositive/perf/query83.q.out ql/src/test/results/clientpositive/perf/query83.q.out index 4f14cc6..dffa986 100644 --- ql/src/test/results/clientpositive/perf/query83.q.out +++ ql/src/test/results/clientpositive/perf/query83.q.out @@ -137,9 +137,9 @@ Reducer 10 <- Reducer 9 (SIMPLE_EDGE) Reducer 11 <- Map 27 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE) Reducer 13 <- Reducer 12 (SIMPLE_EDGE) -Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE) -Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE) -Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE) +Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 20 (ONE_TO_ONE_EDGE) +Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 22 (ONE_TO_ONE_EDGE) +Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 24 (ONE_TO_ONE_EDGE) Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 25 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 20 <- Reducer 19 (SIMPLE_EDGE) @@ -196,8 +196,8 @@ Stage-0 predicate:(d_date is not null and d_date_sk is not null) TableScan [TS_6] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"] - <-Reducer 22 [SIMPLE_EDGE] - SHUFFLE [RS_66] + <-Reducer 22 [ONE_TO_ONE_EDGE] + FORWARD [RS_66] PartitionCols:_col0 Group By Operator [GBY_63] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 @@ -290,8 +290,8 @@ Stage-0 SHUFFLE [RS_105] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_8] - <-Reducer 24 [SIMPLE_EDGE] - SHUFFLE [RS_106] + <-Reducer 24 [ONE_TO_ONE_EDGE] + FORWARD [RS_106] PartitionCols:_col0 Group By Operator [GBY_103] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0 @@ -331,8 +331,8 @@ Stage-0 SHUFFLE [RS_25] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_8] - <-Reducer 20 [SIMPLE_EDGE] - SHUFFLE [RS_26] + <-Reducer 20 [ONE_TO_ONE_EDGE] + FORWARD [RS_26] PartitionCols:_col0 Group By Operator [GBY_23] (rows=40176 width=1119) Output:["_col0"],keys:KEY._col0