diff --git a/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out b/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out index a0f5183..2fcf180 100644 --- a/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out +++ b/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out @@ -69,7 +69,7 @@ STAGE PLANS: Processor Tree: TableScan alias: hbase_ck_4 - filterExpr: ((key.col1 = '238') and (key.col2 = '1238')) (type: boolean) + filterExpr: ((key.col2 = '1238') and (key.col1 = '238')) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: ((key.col1 = '238') and (key.col2 = '1238')) (type: boolean) @@ -107,7 +107,7 @@ STAGE PLANS: Processor Tree: TableScan alias: hbase_ck_4 - filterExpr: ((key.col1 >= '165') and (key.col1 < '27')) (type: boolean) + filterExpr: ((key.col1 < '27') and (key.col1 >= '165')) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: ((key.col1 >= '165') and (key.col1 < '27')) (type: boolean) @@ -146,7 +146,7 @@ STAGE PLANS: Processor Tree: TableScan alias: hbase_ck_4 - filterExpr: ((key.col1 > '100') and (key.col2 >= '1238')) (type: boolean) + filterExpr: ((key.col2 >= '1238') and (key.col1 > '100')) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: ((key.col1 > '100') and (key.col2 >= '1238')) (type: boolean) diff --git a/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out b/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out index 25032cc..e6c307c 100644 --- a/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out +++ b/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out @@ -69,7 +69,7 @@ STAGE PLANS: Processor Tree: TableScan alias: hbase_ck_5 - filterExpr: ((key.col1 = '238') and (key.col2 = '1238')) (type: boolean) + filterExpr: ((key.col2 = '1238') and (key.col1 = '238')) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: ((key.col1 = '238') and (key.col2 = '1238')) (type: boolean) diff --git a/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out b/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out index 97e9aa2..108a975 100644 --- a/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out +++ b/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out @@ -358,7 +358,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hbase_pushdown - filterExpr: ((key < '80') and (key > '90')) (type: boolean) + filterExpr: ((key > '90') and (key < '80')) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: (value like '%90%') (type: boolean) @@ -522,7 +522,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hbase_pushdown - filterExpr: ((key > '281') and (key < '287')) (type: boolean) + filterExpr: ((key < '287') and (key > '281')) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: ((key > '281') and (key < '287')) (type: boolean) diff --git a/hbase-handler/src/test/results/positive/hbase_timestamp.q.out b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out index 7aef504..2d5d14e 100644 --- a/hbase-handler/src/test/results/positive/hbase_timestamp.q.out +++ b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out @@ -171,7 +171,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hbase_table - filterExpr: (((key > 100.0) and (key < 400.0)) and (time < 200000000000)) (type: boolean) + filterExpr: (((time < 200000000000) and (key < 400.0)) and (key > 100.0)) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time < 200000000000))) (type: boolean) @@ -220,7 +220,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hbase_table - filterExpr: (((key > 100.0) and (key < 400.0)) and (time > 100000000000)) (type: boolean) + filterExpr: (((time > 100000000000) and (key < 400.0)) and (key > 100.0)) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time > 100000000000))) (type: boolean) @@ -271,7 +271,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hbase_table - filterExpr: (((key > 100.0) and (key < 400.0)) and (time <= 100000000000)) (type: boolean) + filterExpr: (((time <= 100000000000) and (key < 400.0)) and (key > 100.0)) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time <= 100000000000))) (type: boolean) @@ -320,7 +320,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hbase_table - filterExpr: (((key > 100.0) and (key < 400.0)) and (time >= 200000000000)) (type: boolean) + filterExpr: (((time >= 200000000000) and (key < 400.0)) and (key > 100.0)) (type: boolean) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time >= 200000000000))) (type: boolean) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java b/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java index cf9131d..16bf113 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.lib; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.IdentityHashMap; @@ -37,7 +36,7 @@ public class DefaultGraphWalker implements GraphWalker { protected Stack opStack; - protected final List toWalk = new ArrayList(); + protected final OptimizedStack toWalk = new OptimizedStack(); protected final IdentityHashMap retMap = new IdentityHashMap(); protected final Dispatcher dispatcher; @@ -104,40 +103,52 @@ public void dispatch(Node nd, Stack ndStack) throws SemanticException { */ public void startWalking(Collection startNodes, HashMap nodeOutput) throws SemanticException { - toWalk.addAll(startNodes); - while (toWalk.size() > 0) { - Node nd = toWalk.remove(0); - walk(nd); - if (nodeOutput != null) { - nodeOutput.put(nd, retMap.get(nd)); + for (Node currStartNode : startNodes) { + // The current node is already done, continue the loop. + if (getDispatchedList().contains(currStartNode)) { + continue; + } + // Push the current node to be walked. + toWalk.push(currStartNode); + // Do a postorder DFS with the currStartNode as the origin. + while (!toWalk.empty()) { + Node currWalkNode = toWalk.peek(); + walk(currWalkNode); + // If we have dispatched the current node, add the return value to the output. + if (nodeOutput != null) { + nodeOutput.put(currWalkNode, retMap.get(currWalkNode)); + } } } } /** - * walk the current operator and its descendants. + * walk the current operator and its descendants in a postorder fashion. * * @param nd * current operator in the graph * @throws SemanticException */ public void walk(Node nd) throws SemanticException { + // The current node is part of the operator stack. if (opStack.empty() || nd != opStack.peek()) { opStack.push(nd); } - - if ((nd.getChildren() == null) - || getDispatchedList().containsAll(nd.getChildren())) { - // all children are done or no need to walk the children + // We can dispatch the current node if : + // 1. The current node does not have any children OR + // 2. Children would have been dispatched since + // we traverse in post-order fashion. + if (nd.getChildren() == null || + getDispatchedList().containsAll(nd.getChildren())) { if (!getDispatchedList().contains(nd)) { dispatch(nd, opStack); } opStack.pop(); + toWalk.remove(nd); return; } - // add children, self to the front of the queue in that order - getToWalk().add(0, nd); - getToWalk().removeAll(nd.getChildren()); - getToWalk().addAll(0, nd.getChildren()); + // Add the children to the beginning of toWalk stack. + toWalk.removeAll(nd.getChildren()); + toWalk.addAll(nd.getChildren()); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lib/ForwardWalker.java b/ql/src/java/org/apache/hadoop/hive/ql/lib/ForwardWalker.java index a2db3b5..4c607d5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lib/ForwardWalker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lib/ForwardWalker.java @@ -18,6 +18,10 @@ package org.apache.hadoop.hive.ql.lib; +import java.util.Collection; +import java.util.HashMap; +import java.util.Set; + import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -43,8 +47,9 @@ protected boolean allParentsDispatched(Node nd) { if (op.getParentOperators() == null) { return true; } + Set dispatchedList = getDispatchedList(); for (Node pNode : op.getParentOperators()) { - if (!getDispatchedList().contains(pNode)) { + if (!dispatchedList.contains(pNode)) { return false; } } @@ -52,14 +57,32 @@ protected boolean allParentsDispatched(Node nd) { } @SuppressWarnings("unchecked") - protected void addAllParents(Node nd) { + protected void addAllUndispatchedParents(Node nd) { Operator op = (Operator) nd; - getToWalk().removeAll(op.getParentOperators()); - getToWalk().addAll(0, op.getParentOperators()); + Set dispatchedList = getDispatchedList(); + for (Node pNode : op.getParentOperators()) { + if (!dispatchedList.contains(pNode)) { + toWalk.push(pNode); + } + } } - /** -* walk the current operator and its descendants. + protected void addAllUndispatchedChildren(Node nd) { + if (nd.getChildren() == null) { + return; + } + Set dispatchedList = getDispatchedList(); + for (Node cNode : nd.getChildren()) { + if (!dispatchedList.contains(cNode)) { + toWalk.push(cNode); + } + } + } + +/** +* walk the current operator and its descendants in the following fashion: +* First the undispatched parent nodes, followed by current operator(if undispatched), +* undispatched children. * * @param nd * current operator in the graph @@ -70,17 +93,21 @@ public void walk(Node nd) throws SemanticException { if (opStack.empty() || nd != opStack.peek()) { opStack.push(nd); } + // If the parent nodes have all been dispatched, we can dispatch nd. if (allParentsDispatched(nd)) { - // all children are done or no need to walk the children + // If the current node is not dispatched yet, dispatch the current node and + // add the children to the toWalk stack. if (!getDispatchedList().contains(nd)) { - getToWalk().addAll(nd.getChildren()); dispatch(nd, opStack); + addAllUndispatchedChildren(nd); } + // We are done with the current node. + toWalk.remove(nd); opStack.pop(); return; } - // add children, self to the front of the queue in that order - getToWalk().add(0, nd); - addAllParents(nd); + // All the parents have not been dispatched yet, hence add the undispatched parents + // to the toWalk stack. + addAllUndispatchedParents(nd); } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lib/OptimizedStack.java b/ql/src/java/org/apache/hadoop/hive/ql/lib/OptimizedStack.java new file mode 100644 index 0000000..607cfba --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/lib/OptimizedStack.java @@ -0,0 +1,29 @@ +package org.apache.hadoop.hive.ql.lib; +import java.util.*; + +public class OptimizedStack extends Stack { + private static final long serialVersionUID = 1986881027644228873L; + + @Override + public boolean removeAll(Collection c) { + int oldHi=0, newHi=0, top=0; + for (int i=0; i 0) { + System.arraycopy(elementData, newHi, elementData, top, elementCount - newHi); + elementCount += top - newHi; + return true; + } else { + return false; + } + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lib/PreOrderWalker.java b/ql/src/java/org/apache/hadoop/hive/ql/lib/PreOrderWalker.java index f22694b..fb62302 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lib/PreOrderWalker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lib/PreOrderWalker.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hive.ql.lib; +import java.util.Collection; +import java.util.HashMap; + import org.apache.hadoop.hive.ql.exec.ConditionalTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -44,6 +47,27 @@ public PreOrderWalker(Dispatcher disp) { } /** + * starting point for walking. + * + * @throws SemanticException + */ + @Override + public void startWalking(Collection startNodes, + HashMap nodeOutput) throws SemanticException { + for (Node currStartNode : startNodes) { + // The current node is already done, continue the loop. + if (getDispatchedList().contains(currStartNode)) { + continue; + } + walk(currStartNode); + // If we have dispatched the current node, add the return value to the output. + if (nodeOutput != null) { + nodeOutput.put(currStartNode, retMap.get(currStartNode)); + } + } + } + + /** * Walk the current operator and its descendants. * * @param nd diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java index 9a45458..ebe49b0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java @@ -162,23 +162,22 @@ public void walk(Node nd) throws SemanticException { if ((child instanceof FileSinkOperator) || (child instanceof ScriptOperator)) { walkChildren = false; + break; } } } - if ((nd.getChildren() == null) - || getDispatchedList().containsAll(nd.getChildren()) || !walkChildren) { + if ((nd.getChildren() == null) || + getDispatchedList().containsAll(nd.getChildren()) || !walkChildren) { // all children are done or no need to walk the children dispatch(nd, opStack); opStack.pop(); + toWalk.remove(nd); return; } - // move all the children to the front of queue - getToWalk().removeAll(nd.getChildren()); - getToWalk().addAll(0, nd.getChildren()); - // add self to the end of the queue - getToWalk().add(nd); - opStack.pop(); + // If we have not visited the child, add it to the toWalk stack. + toWalk.removeAll(nd.getChildren()); + toWalk.addAll(nd.getChildren()); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java index dd53ced..181eb33 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; +import org.apache.hadoop.hive.ql.lib.ForwardWalker; import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; @@ -133,37 +134,11 @@ public ParseContext transform(ParseContext pactx) throws SemanticException { /** * Walks the op tree in root first order. */ - public static class ConstantPropagateWalker extends DefaultGraphWalker { + public static class ConstantPropagateWalker extends ForwardWalker { public ConstantPropagateWalker(Dispatcher disp) { super(disp); } - - @Override - public void walk(Node nd) throws SemanticException { - - List parents = ((Operator) nd).getParentOperators(); - if ((parents == null) - || getDispatchedList().containsAll(parents)) { - opStack.push(nd); - - // all children are done or no need to walk the children - dispatch(nd, opStack); - opStack.pop(); - } else { - getToWalk().removeAll(parents); - getToWalk().add(0, nd); - getToWalk().addAll(0, parents); - return; - } - - // move all the children to the front of queue - List children = nd.getChildren(); - if (children != null) { - getToWalk().removeAll(children); - getToWalk().addAll(children); - } - } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java index 9583a1b..58e9c15 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse; +import java.util.Collection; +import java.util.HashMap; import java.util.List; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; @@ -40,6 +42,23 @@ public GenMapRedWalker(Dispatcher disp) { } /** + * starting point for walking. + * + * @throws SemanticException + */ + @Override + public void startWalking(Collection startNodes, + HashMap nodeOutput) throws SemanticException { + for (Node currStartNode : startNodes) { + walk(currStartNode); + // If we have dispatched the current node, add the return value to the output. + if (nodeOutput != null) { + nodeOutput.put(currStartNode, retMap.get(currStartNode)); + } + } + } + + /** * Walk the given operator. * * @param nd diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWorkWalker.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWorkWalker.java index 2d8c8b2..37943c6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWorkWalker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWorkWalker.java @@ -65,9 +65,7 @@ private void setRoot(Node nd) { @Override public void startWalking(Collection startNodes, HashMap nodeOutput) throws SemanticException { - toWalk.addAll(startNodes); - while (toWalk.size() > 0) { - Node nd = toWalk.remove(0); + for (Node nd : startNodes) { setRoot(nd); walk(nd); if (nodeOutput != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezWalker.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezWalker.java index 2f63c1a..96ef037 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezWalker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezWalker.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse; +import java.util.Collection; +import java.util.HashMap; import java.util.List; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; @@ -40,6 +42,23 @@ public TezWalker(Dispatcher disp) { } /** + * starting point for walking. + * + * @throws SemanticException + */ + @Override + public void startWalking(Collection startNodes, + HashMap nodeOutput) throws SemanticException { + for (Node currStartNode : startNodes) { + walk(currStartNode); + // If we have dispatched the current node, add the return value to the output. + if (nodeOutput != null) { + nodeOutput.put(currStartNode, retMap.get(currStartNode)); + } + } + } + + /** * Walk the given operator. * * @param nd diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java index e31c025..3f90c35 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java @@ -65,9 +65,7 @@ private void setRoot(Node nd) { @Override public void startWalking(Collection startNodes, HashMap nodeOutput) throws SemanticException { - toWalk.addAll(startNodes); - while (toWalk.size() > 0) { - Node nd = toWalk.remove(0); + for (Node nd : startNodes) { setRoot(nd); walk(nd); if (nodeOutput != null) { diff --git a/ql/src/test/results/clientpositive/index_auto.q.out b/ql/src/test/results/clientpositive/index_auto.q.out index 28f1e96..751c5c2 100644 --- a/ql/src/test/results/clientpositive/index_auto.q.out +++ b/ql/src/test/results/clientpositive/index_auto.q.out @@ -147,9 +147,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_src_index__ - filterExpr: ((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean) + filterExpr: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) (type: boolean) Filter Operator - predicate: ((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean) + predicate: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) (type: boolean) Select Operator expressions: _bucketname (type: string), _offsets (type: array) outputColumnNames: _col0, _col1 diff --git a/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out b/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out index 7755ed5..bf923c1 100644 --- a/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out +++ b/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out @@ -224,9 +224,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__srcpart_srcpart_index__ - filterExpr: (((((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) + filterExpr: (((((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 70.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Filter Operator - predicate: (((((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) + predicate: (((((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 70.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Select Operator expressions: _bucketname (type: string), _offset (type: bigint) outputColumnNames: _col0, _col1 @@ -321,9 +321,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_src_index__ - filterExpr: (((((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 90.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) + filterExpr: (((((UDFToDouble(key) < 90.0) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 80.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Filter Operator - predicate: (((((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 90.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) + predicate: (((((UDFToDouble(key) < 90.0) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 80.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Select Operator expressions: _bucketname (type: string), _offset (type: bigint) outputColumnNames: _col0, _col1 diff --git a/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out b/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out index 1aa82ba..14407f4 100644 --- a/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out +++ b/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out @@ -231,9 +231,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__srcpart_srcpart_index__ - filterExpr: ((((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 100.0)) (type: boolean) + filterExpr: ((((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 70.0)) (type: boolean) Filter Operator - predicate: ((((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 100.0)) (type: boolean) + predicate: ((((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 70.0)) (type: boolean) Select Operator expressions: _bucketname (type: string), _offsets (type: array) outputColumnNames: _col0, _col1 @@ -349,9 +349,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_src_index__ - filterExpr: ((((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 90.0)) (type: boolean) + filterExpr: ((((UDFToDouble(key) < 90.0) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 80.0)) (type: boolean) Filter Operator - predicate: ((((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 90.0)) (type: boolean) + predicate: ((((UDFToDouble(key) < 90.0) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 80.0)) (type: boolean) Select Operator expressions: _bucketname (type: string), _offsets (type: array) outputColumnNames: _col0, _col1 diff --git a/ql/src/test/results/clientpositive/index_auto_partitioned.q.out b/ql/src/test/results/clientpositive/index_auto_partitioned.q.out index a572471..ddeab4d 100644 --- a/ql/src/test/results/clientpositive/index_auto_partitioned.q.out +++ b/ql/src/test/results/clientpositive/index_auto_partitioned.q.out @@ -64,7 +64,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__srcpart_src_part_index__ - filterExpr: ((UDFToDouble(key) = 86.0) and (ds = '2008-04-09')) (type: boolean) + filterExpr: ((ds = '2008-04-09') and (UDFToDouble(key) = 86.0)) (type: boolean) Filter Operator predicate: (UDFToDouble(key) = 86.0) (type: boolean) Select Operator diff --git a/ql/src/test/results/clientpositive/index_auto_self_join.q.out b/ql/src/test/results/clientpositive/index_auto_self_join.q.out index a214845..f60eb3e 100644 --- a/ql/src/test/results/clientpositive/index_auto_self_join.q.out +++ b/ql/src/test/results/clientpositive/index_auto_self_join.q.out @@ -132,9 +132,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_src_index__ - filterExpr: (((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) + filterExpr: (((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Filter Operator - predicate: (((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) + predicate: (((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Select Operator expressions: _bucketname (type: string), _offset (type: bigint) outputColumnNames: _col0, _col1 @@ -230,9 +230,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_src_index__ - filterExpr: (((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) + filterExpr: (((UDFToDouble(key) < 90.0) and (UDFToDouble(key) > 70.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Filter Operator - predicate: (((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) + predicate: (((UDFToDouble(key) < 90.0) and (UDFToDouble(key) > 70.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Select Operator expressions: _bucketname (type: string), _offset (type: bigint) outputColumnNames: _col0, _col1 diff --git a/ql/src/test/results/clientpositive/index_bitmap_compression.q.out b/ql/src/test/results/clientpositive/index_bitmap_compression.q.out index 9673f08..e0cbf79 100644 --- a/ql/src/test/results/clientpositive/index_bitmap_compression.q.out +++ b/ql/src/test/results/clientpositive/index_bitmap_compression.q.out @@ -39,9 +39,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_src_index__ - filterExpr: (((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) + filterExpr: (((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Filter Operator - predicate: (((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) + predicate: (((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean) Select Operator expressions: _bucketname (type: string), _offset (type: bigint) outputColumnNames: _col0, _col1 diff --git a/ql/src/test/results/clientpositive/index_compression.q.out b/ql/src/test/results/clientpositive/index_compression.q.out index 2ce196b..504fbf4 100644 --- a/ql/src/test/results/clientpositive/index_compression.q.out +++ b/ql/src/test/results/clientpositive/index_compression.q.out @@ -43,9 +43,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_src_index__ - filterExpr: ((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean) + filterExpr: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) (type: boolean) Filter Operator - predicate: ((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean) + predicate: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) > 80.0)) (type: boolean) Select Operator expressions: _bucketname (type: string), _offsets (type: array) outputColumnNames: _col0, _col1 diff --git a/ql/src/test/results/clientpositive/subquery_multiinsert.q.java1.7.out b/ql/src/test/results/clientpositive/subquery_multiinsert.q.java1.7.out index 3a2473f..321777f 100644 --- a/ql/src/test/results/clientpositive/subquery_multiinsert.q.java1.7.out +++ b/ql/src/test/results/clientpositive/subquery_multiinsert.q.java1.7.out @@ -34,7 +34,7 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@src_5 RUN: Stage-0:DDL -Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-4:MAPRED' is a cross product PREHOOK: query: explain from src b INSERT OVERWRITE TABLE src_4 @@ -64,78 +64,96 @@ INSERT OVERWRITE TABLE src_5 order by key POSTHOOK: type: QUERY STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-2, Stage-10 + Stage-5 depends on stages: Stage-4 + Stage-6 depends on stages: Stage-5 + Stage-1 depends on stages: Stage-6 + Stage-7 depends on stages: Stage-1 Stage-10 is a root stage - Stage-2 depends on stages: Stage-10 - Stage-3 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-3 - Stage-1 depends on stages: Stage-4 - Stage-5 depends on stages: Stage-1 - Stage-6 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-6 - Stage-7 depends on stages: Stage-0 STAGE PLANS: - Stage: Stage-10 + Stage: Stage-2 Map Reduce Map Operator Tree: TableScan - alias: s1 + alias: b Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '2') and key is null) (type: boolean) + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Map-reduce partition columns: key (type: string), value (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key > '9') and value is not null) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count() + keys: _col0 (type: string), _col1 (type: string) mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (_col0 = 0) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 0 (type: bigint) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 key (type: string), value (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_4 - Stage: Stage-2 + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_4 + + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: string), value (type: string) - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe TableScan Reduce Output Operator sort order: @@ -156,7 +174,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-3 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -204,7 +222,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-4 + Stage: Stage-6 Map Reduce Map Operator Tree: TableScan @@ -237,70 +255,55 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_5 - Stage: Stage-5 + Stage: Stage-7 Stats-Aggr Operator - Stage: Stage-6 + Stage: Stage-10 Map Reduce Map Operator Tree: TableScan - Reduce Output Operator - key expressions: key (type: string), value (type: string) - sort order: ++ - Map-reduce partition columns: key (type: string), value (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: a + alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > '9') and value is not null) (type: boolean) + predicate: ((key > '2') and key is null) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: string), _col1 (type: string) + aggregations: count() mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) Reduce Operator Tree: - Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 key (type: string), value (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src_4 - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src_4 - - Stage: Stage-7 - Stats-Aggr Operator + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 = 0) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: bigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe -Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-4:MAPRED' is a cross product PREHOOK: query: from src b INSERT OVERWRITE TABLE src_4 select * @@ -337,15 +340,15 @@ POSTHOOK: Lineage: src_4.key EXPRESSION [(src)b.FieldSchema(name:key, type:strin POSTHOOK: Lineage: src_4.value EXPRESSION [(src)b.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: src_5.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src_5.value EXPRESSION [(src)b.FieldSchema(name:value, type:string, comment:default), ] -RUN: Stage-10:MAPRED RUN: Stage-2:MAPRED -RUN: Stage-3:MAPRED -RUN: Stage-6:MAPRED -RUN: Stage-4:MAPRED +RUN: Stage-10:MAPRED RUN: Stage-0:MOVE +RUN: Stage-4:MAPRED +RUN: Stage-3:STATS +RUN: Stage-5:MAPRED +RUN: Stage-6:MAPRED RUN: Stage-1:MOVE RUN: Stage-7:STATS -RUN: Stage-5:STATS PREHOOK: query: select * from src_4 PREHOOK: type: QUERY PREHOOK: Input: default@src_4 @@ -492,8 +495,8 @@ POSTHOOK: Input: default@src_5 199 val_199 199 val_199 2 val_2 -Warning: Map Join MAPJOIN[62][bigTable=b] in task 'Stage-13:MAPRED' is a cross product -Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Map Join MAPJOIN[60][bigTable=?] in task 'Stage-12:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-4:MAPRED' is a cross product PREHOOK: query: explain from src b INSERT OVERWRITE TABLE src_4 @@ -524,18 +527,18 @@ INSERT OVERWRITE TABLE src_5 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-10 is a root stage - Stage-14 depends on stages: Stage-10 , consists of Stage-17, Stage-2 - Stage-17 has a backup stage: Stage-2 - Stage-13 depends on stages: Stage-17 - Stage-15 depends on stages: Stage-2, Stage-13 - Stage-12 depends on stages: Stage-15 - Stage-0 depends on stages: Stage-12 - Stage-7 depends on stages: Stage-0 - Stage-16 depends on stages: Stage-2, Stage-13 - Stage-4 depends on stages: Stage-16 - Stage-1 depends on stages: Stage-4 - Stage-5 depends on stages: Stage-1 - Stage-2 + Stage-13 depends on stages: Stage-10, Stage-14 , consists of Stage-16, Stage-4 + Stage-16 has a backup stage: Stage-4 + Stage-12 depends on stages: Stage-16 + Stage-15 depends on stages: Stage-4, Stage-12 + Stage-6 depends on stages: Stage-15 + Stage-1 depends on stages: Stage-6 + Stage-7 depends on stages: Stage-1 + Stage-4 + Stage-17 is a root stage + Stage-14 depends on stages: Stage-17 + Stage-0 depends on stages: Stage-14 + Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-10 @@ -583,29 +586,27 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-14 + Stage: Stage-13 Conditional Operator - Stage: Stage-17 + Stage: Stage-16 Map Reduce Local Work Alias -> Map Local Tables: - $INTNAME + $INTNAME1 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $INTNAME + $INTNAME1 TableScan HashTable Sink Operator keys: 0 1 - Stage: Stage-13 + Stage: Stage-12 Map Reduce Map Operator Tree: TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Semi Join 0 to 1 @@ -620,82 +621,12 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work Stage: Stage-15 Map Reduce Local Work Alias -> Map Local Tables: - sq_1:a - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - sq_1:a - TableScan - alias: a - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((key > '9') and value is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 key (type: string), value (type: string) - 1 _col0 (type: string), _col1 (type: string) - - Stage: Stage-12 - Map Reduce - Map Operator Tree: - TableScan - Map Join Operator - condition map: - Left Semi Join 0 to 1 - keys: - 0 key (type: string), value (type: string) - 1 _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src_4 - Local Work: - Map Reduce Local Work - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src_4 - - Stage: Stage-7 - Stats-Aggr Operator - - Stage: Stage-16 - Map Reduce Local Work - Alias -> Map Local Tables: sq_2:s1 Fetch Operator limit: -1 @@ -716,7 +647,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) - Stage: Stage-4 + Stage: Stage-6 Map Reduce Map Operator Tree: TableScan @@ -766,25 +697,17 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_5 - Stage: Stage-5 + Stage: Stage-7 Stats-Aggr Operator - Stage: Stage-2 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: string), value (type: string) - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe TableScan Reduce Output Operator sort order: @@ -805,8 +728,83 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe -Warning: Map Join MAPJOIN[62][bigTable=b] in task 'Stage-13:MAPRED' is a cross product -Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product + Stage: Stage-17 + Map Reduce Local Work + Alias -> Map Local Tables: + sq_1:a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + sq_1:a + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key > '9') and value is not null) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 key (type: string), value (type: string) + 1 _col0 (type: string), _col1 (type: string) + + Stage: Stage-14 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 key (type: string), value (type: string) + 1 _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src_4 + + Stage: Stage-3 + Stats-Aggr Operator + +Warning: Map Join MAPJOIN[60][bigTable=?] in task 'Stage-12:MAPRED' is a cross product +Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-4:MAPRED' is a cross product PREHOOK: query: from src b INSERT OVERWRITE TABLE src_4 select * @@ -844,17 +842,17 @@ POSTHOOK: Lineage: src_4.value EXPRESSION [(src)b.FieldSchema(name:value, type:s POSTHOOK: Lineage: src_5.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: src_5.value EXPRESSION [(src)b.FieldSchema(name:value, type:string, comment:default), ] RUN: Stage-10:MAPRED -RUN: Stage-14:CONDITIONAL RUN: Stage-17:MAPREDLOCAL -RUN: Stage-13:MAPRED -RUN: Stage-15:MAPREDLOCAL +RUN: Stage-14:MAPRED +RUN: Stage-0:MOVE +RUN: Stage-13:CONDITIONAL +RUN: Stage-3:STATS RUN: Stage-16:MAPREDLOCAL RUN: Stage-12:MAPRED -RUN: Stage-4:MAPRED -RUN: Stage-0:MOVE +RUN: Stage-15:MAPREDLOCAL +RUN: Stage-6:MAPRED RUN: Stage-1:MOVE RUN: Stage-7:STATS -RUN: Stage-5:STATS PREHOOK: query: select * from src_4 PREHOOK: type: QUERY PREHOOK: Input: default@src_4 diff --git a/ql/src/test/results/clientpositive/table_access_keys_stats.q.out b/ql/src/test/results/clientpositive/table_access_keys_stats.q.out index 7576b48..1f84fe0 100644 --- a/ql/src/test/results/clientpositive/table_access_keys_stats.q.out +++ b/ql/src/test/results/clientpositive/table_access_keys_stats.q.out @@ -260,11 +260,11 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Output: default@t2 PREHOOK: Output: default@t3 -Operator:GBY_2 +Operator:GBY_8 Table:default@t1 Keys:key -Operator:GBY_8 +Operator:GBY_2 Table:default@t1 Keys:key diff --git a/ql/src/test/results/clientpositive/union_view.q.out b/ql/src/test/results/clientpositive/union_view.q.out index c55dee6..090838d 100644 --- a/ql/src/test/results/clientpositive/union_view.q.out +++ b/ql/src/test/results/clientpositive/union_view.q.out @@ -327,7 +327,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_union_1_src_union_1_key_idx__ - filterExpr: ((key = 86) and (ds = '1')) (type: boolean) + filterExpr: ((ds = '1') and (key = 86)) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) Select Operator @@ -440,7 +440,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_union_2_src_union_2_key_idx__ - filterExpr: ((key = 86) and (ds = '2')) (type: boolean) + filterExpr: ((ds = '2') and (key = 86)) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) Select Operator @@ -553,7 +553,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_union_3_src_union_3_key_idx__ - filterExpr: ((key = 86) and (ds = '3')) (type: boolean) + filterExpr: ((ds = '3') and (key = 86)) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) Select Operator @@ -1213,7 +1213,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_union_3_src_union_3_key_idx__ - filterExpr: ((key = 86) and (ds = '4')) (type: boolean) + filterExpr: ((ds = '4') and (key = 86)) (type: boolean) Filter Operator predicate: (key = 86) (type: boolean) Select Operator