diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java b/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java index e45b5a98d2..310dee39d4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java @@ -89,6 +89,40 @@ public void dispatch(Node nd, Stack ndStack) throws SemanticException { dispatchAndReturn(nd, ndStack); } + private class NodeLabeled { + private Node nd; + private int currChildIdx; + + NodeLabeled(Node nd) { + this.nd = nd; + this.currChildIdx = -1; + } + + public void incrementChildIdx() { + this.currChildIdx++; + } + + public int getCurrChildIdx() { + return this.currChildIdx; + } + + public Node getNd() { + return this.nd; + } + } + + /** + * If childNode should be bypassed and not dispatched. + * Default behavior is to not bypass any node + * Graphers can override this functionality, e.g. ExpressionWalker + * @param childNode + * @param parentNode + * @return + */ + protected boolean shouldByPass(Node childNode, Node parentNode) { + return false; + } + /** * Returns dispatch result */ @@ -144,32 +178,34 @@ public void startWalking(Collection startNodes, * @throws SemanticException */ protected void walk(Node nd) throws SemanticException { - // Push the node in the stack - opStack.push(nd); + Stack traversalStack = new Stack<>(); + traversalStack.push(new NodeLabeled(nd)); - // While there are still nodes to dispatch... - while (!opStack.empty()) { - Node node = opStack.peek(); + opStack.push(nd); - if (node.getChildren() == null || - getDispatchedList().containsAll(node.getChildren())) { - // Dispatch current node - if (!getDispatchedList().contains(node)) { - dispatch(node, opStack); - opQueue.add(node); + while(!traversalStack.isEmpty()) { + NodeLabeled currLabeledNode = traversalStack.peek(); + Node currNode = currLabeledNode.getNd(); + int currIdx = currLabeledNode.getCurrChildIdx(); + + if(currNode.getChildren() != null && currNode.getChildren().size() > currIdx + 1) { + Node nextChild = currNode.getChildren().get(currIdx+1); + //check if this node should be skipped and not dispatched + if(shouldByPass(nextChild, currNode)) { + retMap.put(nextChild, null); + currLabeledNode.incrementChildIdx(); + continue; } + traversalStack.push(new NodeLabeled(nextChild)); + opStack.push(nextChild); + currLabeledNode.incrementChildIdx(); + } else { + // dispatch the node + dispatch(currNode, opStack); + opQueue.add(currNode); opStack.pop(); - continue; + traversalStack.pop(); } - - // Add a single child and restart the loop - for (Node childNode : node.getChildren()) { - if (!getDispatchedList().contains(childNode)) { - opStack.push(childNode); - break; - } - } - } // end while + } } - } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lib/ExpressionWalker.java b/ql/src/java/org/apache/hadoop/hive/ql/lib/ExpressionWalker.java index 41607aeec5..37ae57e454 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lib/ExpressionWalker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lib/ExpressionWalker.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; -import org.apache.hadoop.hive.ql.parse.SemanticException; public class ExpressionWalker extends DefaultGraphWalker { @@ -40,7 +39,7 @@ public ExpressionWalker(Dispatcher disp) { * (in genLogicalPlan) for subquery at this point. * SubQueryExprProcessor will use generated plan and creates appropriate ExprNodeSubQueryDesc. */ - private boolean shouldByPass(Node childNode, Node parentNode) { + protected boolean shouldByPass(Node childNode, Node parentNode) { if(parentNode instanceof ASTNode && ((ASTNode)parentNode).getType() == HiveParser.TOK_SUBQUERY_EXPR) { ASTNode parentOp = (ASTNode)parentNode; @@ -54,44 +53,6 @@ private boolean shouldByPass(Node childNode, Node parentNode) { } return false; } - /** - * walk the current operator and its descendants. - * - * @param nd - * current operator in the graph - * @throws SemanticException - */ - protected void walk(Node nd) throws SemanticException { - // Push the node in the stack - opStack.push(nd); - - // While there are still nodes to dispatch... - while (!opStack.empty()) { - Node node = opStack.peek(); - if (node.getChildren() == null || - getDispatchedList().containsAll(node.getChildren())) { - // Dispatch current node - if (!getDispatchedList().contains(node)) { - dispatch(node, opStack); - opQueue.add(node); - } - opStack.pop(); - continue; - } - - // Add a single child and restart the loop - for (Node childNode : node.getChildren()) { - if (!getDispatchedList().contains(childNode)) { - if(shouldByPass(childNode, node)) { - retMap.put(childNode, null); - } else { - opStack.push(childNode); - } - break; - } - } - } // end while - } } diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index e17ac7bffd..4837deb2b4 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -2459,12 +2459,12 @@ Stage-0 Stage-1 Reducer 5 llap File Output Operator [FS_33] - Merge Join Operator [MERGEJOIN_56] (rows=41 width=186) + Merge Join Operator [MERGEJOIN_57] (rows=41 width=186) Conds:RS_29._col2=RS_30._col0(Left Semi),Output:["_col0","_col1","_col2"] <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_29] PartitionCols:_col2 - Filter Operator [FIL_39] (rows=65 width=186) + Filter Operator [FIL_41] (rows=65 width=186) predicate:_col2 is not null Group By Operator [GBY_16] (rows=65 width=186) Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1 @@ -2473,14 +2473,14 @@ Stage-0 PartitionCols:_col0, _col1 Group By Operator [GBY_14] (rows=65 width=186) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col2 - Merge Join Operator [MERGEJOIN_55] (rows=131 width=178) + Merge Join Operator [MERGEJOIN_56] (rows=131 width=178) Conds:RS_10._col0=RS_11._col0(Inner),Output:["_col1","_col2"] <-Map 7 [SIMPLE_EDGE] llap SHUFFLE [RS_11] PartitionCols:_col0 Select Operator [SEL_9] (rows=166 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=166 width=178) + Filter Operator [FIL_42] (rows=166 width=178) predicate:(key > '8') TableScan [TS_7] (rows=500 width=178) default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] @@ -2505,9 +2505,9 @@ Stage-0 Output:["_col0"],keys:_col0 Select Operator [SEL_26] (rows=83 width=8) Output:["_col0"] - Filter Operator [FIL_42] (rows=83 width=8) + Filter Operator [FIL_43] (rows=83 width=8) predicate:_col1 is not null - Select Operator [SEL_44] (rows=83 width=8) + Select Operator [SEL_45] (rows=83 width=8) Output:["_col1"] Group By Operator [GBY_24] (rows=83 width=95) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 @@ -2516,7 +2516,7 @@ Stage-0 PartitionCols:_col0 Group By Operator [GBY_22] (rows=83 width=95) Output:["_col0","_col1"],aggregations:["count()"],keys:key - Filter Operator [FIL_43] (rows=166 width=87) + Filter Operator [FIL_44] (rows=166 width=87) predicate:(key > '9') Please refer to the previous TableScan [TS_0]