diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java index b56b608..988bb29 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.PreOrderWalker; +import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -75,7 +75,7 @@ public static RewriteCanApplyCtx getInstance(ParseContext parseContext){ private String baseTableName; private String indexTableName; private String aggFunction; - + private TableScanOperator tableScanOperator; private List selectOperators; private List groupByOperators; @@ -156,7 +156,7 @@ public ParseContext getParseContext() { /** * This method walks all the nodes starting from topOp TableScanOperator node * and invokes methods from {@link RewriteCanApplyProcFactory} for each of the rules - * added to the opRules map. We use the {@link PreOrderWalker} for a pre-order + * added to the opRules map. We use the {@link PreOrderOnceWalker} for a pre-order * traversal of the operator tree. * * The methods from {@link RewriteCanApplyProcFactory} set appropriate values in @@ -180,7 +180,7 @@ void populateRewriteVars(TableScanOperator topOp) // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, this); - GraphWalker ogw = new PreOrderWalker(disp); + GraphWalker ogw = new PreOrderOnceWalker(disp); // Create a list of topop nodes List topNodes = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java index f901812..8dcd283 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; -import org.apache.hadoop.hive.ql.lib.PreOrderWalker; +import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer.WalkerCtx; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -61,7 +61,7 @@ public class NullScanTaskDispatcher implements Dispatcher { static final Log LOG = LogFactory.getLog(NullScanTaskDispatcher.class.getName()); - + private final PhysicalContext physicalContext; private final Map rules; @@ -177,7 +177,7 @@ public int compare(MapWork o1, MapWork o2) { // The dispatcher fires the processor corresponding to the closest // matching rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(null, rules, walkerCtx); - GraphWalker ogw = new PreOrderWalker(disp); + GraphWalker ogw = new PreOrderOnceWalker(disp); // Create a list of topOp nodes ArrayList topNodes = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 7ecd50a..2e3bd76 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.exec.spark.SparkTask; import org.apache.hadoop.hive.ql.exec.tez.TezTask; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyLongOperator; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyMultiKeyOperator; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyStringOperator; @@ -68,6 +67,7 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; @@ -155,7 +155,6 @@ Set supportedAggregationUdfs = new HashSet(); - private PhysicalContext physicalContext = null; private HiveConf hiveConf; public Vectorizer() { @@ -307,13 +306,10 @@ public Vectorizer() { class VectorizationDispatcher implements Dispatcher { - private final PhysicalContext physicalContext; - private List reduceColumnNames; private List reduceTypeInfos; public VectorizationDispatcher(PhysicalContext physicalContext) { - this.physicalContext = physicalContext; reduceColumnNames = null; reduceTypeInfos = null; } @@ -428,7 +424,7 @@ private void vectorizeMapWork(MapWork mapWork, boolean isTez) throws SemanticExc MapWorkVectorizationNodeProcessor vnp = new MapWorkVectorizationNodeProcessor(mapWork, isTez); addMapWorkRules(opRules, vnp); Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null); - GraphWalker ogw = new PreOrderWalker(disp); + GraphWalker ogw = new PreOrderOnceWalker(disp); // iterator the mapper operator tree ArrayList topNodes = new ArrayList(); topNodes.addAll(mapWork.getAliasToWork().values()); @@ -701,12 +697,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, class MapWorkVectorizationNodeProcessor extends VectorizationNodeProcessor { - private final MapWork mWork; private final boolean isTez; public MapWorkVectorizationNodeProcessor(MapWork mWork, boolean isTez) { super(); - this.mWork = mWork; this.isTez = isTez; } @@ -765,7 +759,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, private final List reduceColumnNames; private final List reduceTypeInfos; - private boolean isTez; + private final boolean isTez; private Operator rootVectorOp; @@ -865,7 +859,6 @@ protected int getInputColumnIndex(ExprNodeColumnDesc colExpr) { @Override public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException { - this.physicalContext = physicalContext; hiveConf = physicalContext.getConf(); boolean vectorPath = HiveConf.getBoolVar(hiveConf, @@ -1406,7 +1399,6 @@ private boolean isBigTableOnlyResults(MapJoinDesc desc) { int[] smallTableIndices; int smallTableIndicesSize; - List smallTableExprs = desc.getExprs().get(posSingleVectorMapJoinSmallTable); if (desc.getValueIndices() != null && desc.getValueIndices().get(posSingleVectorMapJoinSmallTable) != null) { smallTableIndices = desc.getValueIndices().get(posSingleVectorMapJoinSmallTable); LOG.info("Vectorizer isBigTableOnlyResults smallTableIndices " + Arrays.toString(smallTableIndices)); @@ -1445,8 +1437,6 @@ private boolean isBigTableOnlyResults(MapJoinDesc desc) { Operator vectorOp = null; Class> opClass = null; - boolean isOuterJoin = !desc.getNoOuterJoin(); - VectorMapJoinDesc.HashTableImplementationType hashTableImplementationType = HashTableImplementationType.NONE; VectorMapJoinDesc.HashTableKind hashTableKind = HashTableKind.NONE; VectorMapJoinDesc.HashTableKeyType hashTableKeyType = HashTableKeyType.NONE; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java index 32af813..44159c5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.PreOrderWalker; +import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.optimizer.Transform; @@ -92,7 +92,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // rule and passes the context along SyntheticContext context = new SyntheticContext(pctx); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context); - GraphWalker ogw = new PreOrderWalker(disp); + GraphWalker ogw = new PreOrderOnceWalker(disp); // Create a list of top op nodes List topNodes = new ArrayList();