diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleExp.java b/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleExp.java new file mode 100644 index 0000000..7ccae31 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleExp.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.lib; + +import java.util.Stack; +import org.apache.hadoop.hive.ql.parse.SemanticException; + +/** + * Rule interface for Nodes Used in Node dispatching to dispatch process/visitor + * functions for Nodes. + */ +public class RuleExp implements Rule { + Rule expRule; + + /** + * The function iterates through the list of wild card characters and sees if + * this contains a wild card character. + */ + private boolean hasWildCardCharacters(String regExp) { + final char[] wildCards = {'[', '^', '*', ']', '+', '|', '(', '\\', '*', ')'}; + + for (char wc : wildCards) { + if (regExp.indexOf(wc) != -1) { + return true; + } + } + return false; + } + + /** + * @param ruleName + * name of the rule + * @param regExp + * regular expression for the rule + **/ + public RuleExp(String ruleName, String regExp) { + expRule = hasWildCardCharacters(regExp) ? new RuleRegExp(ruleName, regExp) : + new RuleStringExp(ruleName, regExp); + } + + /** + * This function returns the cost of the rule for the specified stack. Lower + * the cost, the better the rule is matched + * + * @param stack + * Node stack encountered so far + * @return cost of the function + * @throws SemanticException + */ + @Override + public int cost(Stack stack) throws SemanticException { + return expRule.cost(stack); + } + + /** + * @return the name of the Node + **/ + @Override + public String getName() { + return expRule.getName(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java b/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java index ddc96c2..4740047 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java @@ -26,7 +26,8 @@ /** * Rule interface for Nodes Used in Node dispatching to dispatch process/visitor - * functions for Nodes. + * functions for Nodes when the rule is expressed as a regular expression. + * @see RuleExp */ public class RuleRegExp implements Rule { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleStringExp.java b/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleStringExp.java new file mode 100644 index 0000000..3e9a319 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleStringExp.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.lib; + +import java.util.Stack; + +import org.apache.hadoop.hive.ql.parse.SemanticException; + +/** + * Rule interface for Nodes Used in Node dispatching to dispatch process/visitor + * functions for Nodes when the rule is expressed as a simple string. + * @see RuleExp + */ +public class RuleStringExp implements Rule { + private final String ruleName; + private final String pattern; + + /** + * The rule specified as operator names separated by % symbols, the left side represents the + * bottom of the stack. + * + * E.g. TS%FIL%RS -> means + * TableScan Node followed by Filter followed by ReduceSink in the tree, or, in terms of the + * stack, ReduceSink on top followed by Filter followed by TableScan + * + * @param ruleName + * name of the rule + * @param regExp + * string specification of the rule + **/ + public RuleStringExp(String ruleName, String pattern) { + this.ruleName = ruleName; + this.pattern = pattern; + } + + /** + * This function returns the cost of the rule for the specified stack. Returns 1 if there is + * an suffix string match with the entire stack, otherwise -1 + * + * If any proper substack of the stack matches it will return -1. It only returns 1 if the + * entire stack matches the rule exactly. + * + * @param stack + * Node stack encountered so far + * @return cost of the function + * @throws SemanticException + */ + @Override + public int cost(Stack stack) throws SemanticException { + int numElems = (stack != null ? stack.size() : 0); + String name = new String(); + int patLen = pattern.length(); + + for (int pos = numElems - 1; pos >= 0; pos--) { + name = stack.get(pos).getName() + "%" + name; + if (name.length() >= patLen) { + if (pattern.equals(name)) { + return patLen; + } else { + return -1; + } + } + } + + return -1; + } + + /** + * @return the name of the Node + **/ + @Override + public String getName() { + return ruleName; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java index 6f35b87..fbc9d9f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -57,7 +57,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { new BucketJoinProcCtx(pctx.getConf()); // process map joins with no reducers pattern - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", MapJoinOperator.getOperatorName() + "%"), getBucketMapjoinProc(pctx)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java index a090a5b..3c58430 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java @@ -45,7 +45,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -83,7 +83,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { Map opRules = new LinkedHashMap(); // process reduce sink added by hive.enforce.bucketing or hive.enforce.sorting - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", ReduceSinkOperator.getOperatorName() + "%" + SelectOperator.getOperatorName() + "%" + FileSinkOperator.getOperatorName() + "%"), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java index b8f5c71..e917799 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -83,40 +83,40 @@ public ParseContext transform(ParseContext pactx) throws SemanticException { // the operator stack. The dispatcher // generates the plan from the operator tree Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", FilterOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getFilterProc()); - opRules.put(new RuleRegExp("R2", + opRules.put(new RuleExp("R2", GroupByOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getGroupByProc()); - opRules.put(new RuleRegExp("R3", + opRules.put(new RuleExp("R3", ReduceSinkOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getReduceSinkProc()); - opRules.put(new RuleRegExp("R4", + opRules.put(new RuleExp("R4", SelectOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getSelectProc()); - opRules.put(new RuleRegExp("R5", + opRules.put(new RuleExp("R5", CommonJoinOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getJoinProc()); - opRules.put(new RuleRegExp("R6", + opRules.put(new RuleExp("R6", MapJoinOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getMapJoinProc()); - opRules.put(new RuleRegExp("R7", + opRules.put(new RuleExp("R7", TableScanOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getTableScanProc()); - opRules.put(new RuleRegExp("R8", + opRules.put(new RuleExp("R8", LateralViewJoinOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getLateralViewJoinProc()); - opRules.put(new RuleRegExp("R9", + opRules.put(new RuleExp("R9", LateralViewForwardOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getLateralViewForwardProc()); - opRules.put(new RuleRegExp("R10", + opRules.put(new RuleExp("R10", PTFOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getPTFProc()); - opRules.put(new RuleRegExp("R11", + opRules.put(new RuleExp("R11", ScriptOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getScriptProc()); - opRules.put(new RuleRegExp("R12", + opRules.put(new RuleExp("R12", LimitOperator.getOperatorName() + "%"), ColumnPrunerProcFactory.getLimitProc()); // The dispatcher fires the processor corresponding to the closest matching diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java index b5ee4ef..8d1ebbb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -91,21 +91,21 @@ public ParseContext transform(ParseContext pactx) throws SemanticException { // generates the plan from the operator tree Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", FilterOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R1", FilterOperator.getOperatorName() + "%"), ConstantPropagateProcFactory.getFilterProc()); - opRules.put(new RuleRegExp("R2", GroupByOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R2", GroupByOperator.getOperatorName() + "%"), ConstantPropagateProcFactory.getGroupByProc()); - opRules.put(new RuleRegExp("R3", SelectOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R3", SelectOperator.getOperatorName() + "%"), ConstantPropagateProcFactory.getSelectProc()); - opRules.put(new RuleRegExp("R4", FileSinkOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R4", FileSinkOperator.getOperatorName() + "%"), ConstantPropagateProcFactory.getFileSinkProc()); - opRules.put(new RuleRegExp("R5", ReduceSinkOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R5", ReduceSinkOperator.getOperatorName() + "%"), ConstantPropagateProcFactory.getReduceSinkProc()); - opRules.put(new RuleRegExp("R6", JoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R6", JoinOperator.getOperatorName() + "%"), ConstantPropagateProcFactory.getJoinProc()); - opRules.put(new RuleRegExp("R7", TableScanOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R7", TableScanOperator.getOperatorName() + "%"), ConstantPropagateProcFactory.getTableScanProc()); - opRules.put(new RuleRegExp("R8", ScriptOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R8", ScriptOperator.getOperatorName() + "%"), ConstantPropagateProcFactory.getStopProc()); // The dispatcher fires the processor corresponding to the closest matching diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java index 8546d21..51729ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java @@ -45,7 +45,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.OptimizeTezProcContext; @@ -328,7 +328,7 @@ private void generateEventOperatorPlan(DynamicListContext ctx, ParseContext pars // the operator stack. The dispatcher // generates the plan from the operator tree Map exprRules = new LinkedHashMap(); - exprRules.put(new RuleRegExp("R1", ExprNodeDynamicListDesc.class.getName() + "%"), + exprRules.put(new RuleExp("R1", ExprNodeDynamicListDesc.class.getName() + "%"), new DynamicPartitionPrunerProc()); // The dispatcher fires the processor corresponding to the closest matching diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java index af54286..645d372 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java @@ -45,7 +45,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -81,14 +81,14 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEGROUPBYSKEW)) { // process group-by pattern - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", GroupByOperator.getOperatorName() + "%" + ReduceSinkOperator.getOperatorName() + "%" + GroupByOperator.getOperatorName() + "%"), getMapSortedGroupbyProc(pctx)); } else { // If hive.groupby.skewindata is set to true, the operator tree is as below - opRules.put(new RuleRegExp("R2", + opRules.put(new RuleExp("R2", GroupByOperator.getOperatorName() + "%" + ReduceSinkOperator.getOperatorName() + "%" + GroupByOperator.getOperatorName() + "%" + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java index e3d3ce6..d044369 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java @@ -41,7 +41,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -82,7 +82,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // 1. We apply the transformation Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", "(" + SelectOperator.getOperatorName() + "%)"), new ProjectRemover()); GraphWalker ogw = new DefaultGraphWalker(new DefaultRuleDispatcher(null, opRules, null)); ArrayList topNodes = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java index e850550..fdacfb1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -90,7 +90,7 @@ @Override public ParseContext transform(ParseContext pctx) throws SemanticException { Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", ReduceSinkOperator.getOperatorName() + "%" + ".*" + LimitOperator.getOperatorName() + "%"), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java index 4d84f0f..560f4a9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java @@ -55,7 +55,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.GenMapRedWalker; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -690,16 +690,16 @@ public ParseContext transform(ParseContext pactx) throws SemanticException { // the operator stack. // The dispatcher generates the plan from the operator tree Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R0", + opRules.put(new RuleExp("R0", MapJoinOperator.getOperatorName() + "%"), getCurrentMapJoin()); - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", MapJoinOperator.getOperatorName() + "%.*" + FileSinkOperator.getOperatorName() + "%"), getMapJoinFS()); - opRules.put(new RuleRegExp("R2", + opRules.put(new RuleExp("R2", MapJoinOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"), getMapJoinDefault()); - opRules.put(new RuleRegExp("R4", + opRules.put(new RuleExp("R4", MapJoinOperator.getOperatorName() + "%.*" + UnionOperator.getOperatorName() + "%"), getMapJoinDefault()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java index 2764ae1..f44436b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -62,8 +62,8 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { String SEL = SelectOperator.getOperatorName(); String FIL = FilterOperator.getOperatorName(); Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", SEL + "%" + SEL + "%"), new SelectDedup(pctx)); - opRules.put(new RuleRegExp("R2", FIL + "%" + FIL + "%"), new FilterDedup()); + opRules.put(new RuleExp("R1", SEL + "%" + SEL + "%"), new SelectDedup(pctx)); + opRules.put(new RuleExp("R2", FIL + "%" + FIL + "%"), new FilterDedup()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new DefaultGraphWalker(disp); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerUtils.java index 108177e..ea3031c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerUtils.java @@ -35,7 +35,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -76,7 +76,7 @@ public static void walkOperatorTree(ParseContext pctx, NodeProcessorCtx opWalker String tsOprName = TableScanOperator.getOperatorName(); String filtOprName = FilterOperator.getOperatorName(); - opRules.put(new RuleRegExp("R1", new StringBuilder().append("(").append(tsOprName).append("%") + opRules.put(new RuleExp("R1", new StringBuilder().append("(").append(tsOprName).append("%") .append(filtOprName).append("%)|(").append(tsOprName).append("%").append(filtOprName) .append("%").append(filtOprName).append("%)").toString()), filterProc); @@ -111,9 +111,9 @@ public static void walkOperatorTree(ParseContext pctx, NodeProcessorCtx opWalker // the operator stack. The dispatcher // generates the plan from the operator tree Map exprRules = new LinkedHashMap(); - exprRules.put(new RuleRegExp("R1", ExprNodeColumnDesc.class.getName() + "%"), colProc); - exprRules.put(new RuleRegExp("R2", ExprNodeFieldDesc.class.getName() + "%"), fieldProc); - exprRules.put(new RuleRegExp("R5", ExprNodeGenericFuncDesc.class.getName() + "%"), + exprRules.put(new RuleExp("R1", ExprNodeColumnDesc.class.getName() + "%"), colProc); + exprRules.put(new RuleExp("R2", ExprNodeFieldDesc.class.getName() + "%"), fieldProc); + exprRules.put(new RuleExp("R5", ExprNodeGenericFuncDesc.class.getName() + "%"), genFuncProc); // The dispatcher fires the processor corresponding to the closest matching diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java index 37f9473..9a35ddd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -104,7 +104,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { .getOpToSamplePruner()); Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%|" diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java index 39e11a2..a4bfa80 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AggregationDesc; @@ -66,8 +66,8 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { String FS = FileSinkOperator.getOperatorName() + "%"; Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", GBY + RS + GBY + SEL + FS), new SingleGBYProcessor(pctx)); - opRules.put(new RuleRegExp("R2", GBY + RS + GBY + FS), new SingleGBYProcessor(pctx)); + opRules.put(new RuleExp("R1", GBY + RS + GBY + SEL + FS), new SingleGBYProcessor(pctx)); + opRules.put(new RuleExp("R2", GBY + RS + GBY + FS), new SingleGBYProcessor(pctx)); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new DefaultGraphWalker(disp); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java index dc885ab..b72553b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java @@ -46,7 +46,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.QBJoinTree; @@ -658,7 +658,7 @@ private static void replaceAlias(List origin, List cloned, public ParseContext transform(ParseContext pctx) throws SemanticException { Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", "TS%.*RS%JOIN%"), getSkewJoinProc(pctx)); + opRules.put(new RuleExp("R1", "TS%.*RS%JOIN%"), getSkewJoinProc(pctx)); SkewJoinOptProcCtx skewJoinOptProcCtx = new SkewJoinOptProcCtx(pctx); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index 7bcb797..a8a329c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -52,7 +52,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -91,7 +91,7 @@ public ParseContext transform(ParseContext pCtx) throws SemanticException { String FS = FileSinkOperator.getOperatorName() + "%"; - opRules.put(new RuleRegExp("Sorted Dynamic Partition", FS), getSortDynPartProc(pCtx)); + opRules.put(new RuleExp("Sorted Dynamic Partition", FS), getSortDynPartProc(pCtx)); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new DefaultGraphWalker(disp); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java index 51f1b74..72eb637 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -59,7 +59,7 @@ private void getListOfRejectedJoins( // Go through all joins - it should only contain selects and filters between // tablescan and join operators. Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", JoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R1", JoinOperator.getOperatorName() + "%"), getCheckCandidateJoin()); // The dispatcher fires the processor corresponding to the closest matching @@ -88,7 +88,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { Map opRules = new LinkedHashMap(); // go through all map joins and find out all which have enabled bucket map // join. - opRules.put(new RuleRegExp("R1", MapJoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R1", MapJoinOperator.getOperatorName() + "%"), getSortedMergeBucketMapjoinProc(pctx)); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along @@ -96,7 +96,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // There is no need for the user to specify mapjoin for it to be // converted to sort-merge join if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN)) { - opRules.put(new RuleRegExp("R2", "JOIN%"), + opRules.put(new RuleExp("R2", "JOIN%"), getSortedMergeJoinProc(pctx)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index bc8d8f7..3a175fb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -51,7 +51,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -110,9 +110,9 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { String FS = FileSinkOperator.getOperatorName() + "%"; Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", TS + SEL + GBY + RS + GBY + SEL + FS), + opRules.put(new RuleExp("R1", TS + SEL + GBY + RS + GBY + SEL + FS), new MetaDataProcessor(pctx)); - opRules.put(new RuleRegExp("R2", TS + SEL + GBY + RS + GBY + FS), + opRules.put(new RuleExp("R2", TS + SEL + GBY + RS + GBY + FS), new MetaDataProcessor(pctx)); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java index d861682..d354ea8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -70,7 +70,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // 2. Trigger transformation Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", JoinOperator.getOperatorName() + "%"), new JoinAnnotate()); + opRules.put(new RuleExp("R1", JoinOperator.getOperatorName() + "%"), new JoinAnnotate()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new ForwardWalker(disp); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java index c1f1519..a45f2a5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java @@ -56,7 +56,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.MapJoinProcessor; import org.apache.hadoop.hive.ql.optimizer.Transform; @@ -218,7 +218,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // detect correlations CorrelationNodeProcCtx corrCtx = new CorrelationNodeProcCtx(pCtx); Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", ReduceSinkOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R1", ReduceSinkOperator.getOperatorName() + "%"), new CorrelationNodeProc()); Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, corrCtx); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java index b56b608..b2ff2cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -170,7 +170,7 @@ void populateRewriteVars(TableScanOperator topOp) Map opRules = new LinkedHashMap(); //^TS%[(SEL%)|(FIL%)]*GRY%[(FIL%)]*RS%[(FIL%)]*GRY% opRules.put( - new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%[(" + new RuleExp("R1", TableScanOperator.getOperatorName() + "%[(" + SelectOperator.getOperatorName() + "%)|(" + FilterOperator.getOperatorName() + "%)]*" + GroupByOperator.getOperatorName() + "%[" + FilterOperator.getOperatorName() + "%]*" + ReduceSinkOperator.getOperatorName() + "%[" + FilterOperator.getOperatorName() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java index c930b80..efcd071 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -185,12 +185,12 @@ public static Dependency getExprDependency(LineageCtx lctx, // generates the plan from the operator tree Map exprRules = new LinkedHashMap(); exprRules.put( - new RuleRegExp("R1", ExprNodeColumnDesc.class.getName() + "%"), + new RuleExp("R1", ExprNodeColumnDesc.class.getName() + "%"), getColumnProcessor()); exprRules.put( - new RuleRegExp("R2", ExprNodeFieldDesc.class.getName() + "%"), + new RuleExp("R2", ExprNodeFieldDesc.class.getName() + "%"), getFieldProcessor()); - exprRules.put(new RuleRegExp("R3", ExprNodeGenericFuncDesc.class.getName() + exprRules.put(new RuleExp("R3", ExprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor()); // The dispatcher fires the processor corresponding to the closest matching diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java index 51bef04..cdc438e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -63,26 +63,26 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { LineageCtx lCtx = new LineageCtx(pctx); Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R1", TableScanOperator.getOperatorName() + "%"), OpProcFactory.getTSProc()); - opRules.put(new RuleRegExp("R2", ScriptOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R2", ScriptOperator.getOperatorName() + "%"), OpProcFactory.getTransformProc()); - opRules.put(new RuleRegExp("R3", UDTFOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R3", UDTFOperator.getOperatorName() + "%"), OpProcFactory.getTransformProc()); - opRules.put(new RuleRegExp("R4", SelectOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R4", SelectOperator.getOperatorName() + "%"), OpProcFactory.getSelProc()); - opRules.put(new RuleRegExp("R5", GroupByOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R5", GroupByOperator.getOperatorName() + "%"), OpProcFactory.getGroupByProc()); - opRules.put(new RuleRegExp("R6", UnionOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R6", UnionOperator.getOperatorName() + "%"), OpProcFactory.getUnionProc()); - opRules.put(new RuleRegExp("R7", + opRules.put(new RuleExp("R7", CommonJoinOperator.getOperatorName() + "%|" + MapJoinOperator.getOperatorName() + "%"), OpProcFactory.getJoinProc()); - opRules.put(new RuleRegExp("R8", ReduceSinkOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R8", ReduceSinkOperator.getOperatorName() + "%"), OpProcFactory.getReduceSinkProc()); - opRules.put(new RuleRegExp("R9", LateralViewJoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R9", LateralViewJoinOperator.getOperatorName() + "%"), OpProcFactory.getLateralViewJoinProc()); - opRules.put(new RuleRegExp("R10", PTFOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R10", PTFOperator.getOperatorName() + "%"), OpProcFactory.getTransformProc()); // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java index c304e97..c1025c2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.OpTraitsRulesProcFactory; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -58,25 +58,25 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // create a walker which walks the tree in a DFS manner while maintaining the // operator stack. The dispatcher generates the plan from the operator tree Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("TS", TableScanOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("TS", TableScanOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getTableScanRule()); - opRules.put(new RuleRegExp("RS", ReduceSinkOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("RS", ReduceSinkOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getReduceSinkRule()); - opRules.put(new RuleRegExp("JOIN", JoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("JOIN", JoinOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getJoinRule()); - opRules.put(new RuleRegExp("MAPJOIN", MapJoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("MAPJOIN", MapJoinOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule()); - opRules.put(new RuleRegExp("SMB", SMBMapJoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("SMB", SMBMapJoinOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule()); - opRules.put(new RuleRegExp("MUX", MuxOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("MUX", MuxOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule()); - opRules.put(new RuleRegExp("DEMUX", DemuxOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("DEMUX", DemuxOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule()); - opRules.put(new RuleRegExp("UNION", UnionOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("UNION", UnionOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getMultiParentRule()); - opRules.put(new RuleRegExp("GBY", GroupByOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("GBY", GroupByOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getGroupByRule()); - opRules.put(new RuleRegExp("SEL", SelectOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("SEL", SelectOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getSelectRule()); // The dispatcher fires the processor corresponding to the closest matching diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java index cbed375..601bcff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java @@ -35,7 +35,7 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -66,7 +66,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { PcrOpWalkerCtx opWalkerCtx = new PcrOpWalkerCtx(pctx, opToRemove); Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)|(" + TableScanOperator.getOperatorName() + "%" diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java index d5102bc..1469207 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; @@ -437,12 +437,12 @@ public static NodeInfoWrapper walkExprTree( Map exprRules = new LinkedHashMap(); exprRules.put( - new RuleRegExp("R1", ExprNodeColumnDesc.class.getName() + "%"), + new RuleExp("R1", ExprNodeColumnDesc.class.getName() + "%"), getColumnProcessor()); exprRules.put( - new RuleRegExp("R2", ExprNodeFieldDesc.class.getName() + "%"), + new RuleExp("R2", ExprNodeFieldDesc.class.getName() + "%"), getFieldProcessor()); - exprRules.put(new RuleRegExp("R5", ExprNodeGenericFuncDesc.class.getName() + exprRules.put(new RuleExp("R5", ExprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor()); // The dispatcher fires the processor corresponding to the closest matching diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java index f370d4d..519fb28 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleExactMatch; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -94,13 +94,13 @@ private void inferBucketingSorting(List mapRedTasks) throws Semantic boolean disableBucketing = mapRedTask.getWork().getMapWork().getSamplingType() > 0; BucketingSortingCtx bCtx = new BucketingSortingCtx(disableBucketing); - // RuleRegExp rules are used to match operators anywhere in the tree + // RuleExp rules are used to match operators anywhere in the tree // RuleExactMatch rules are used to specify exactly what the tree should look like // In particular, this guarantees that the first operator is the reducer // (and its parent(s) are ReduceSinkOperators) since it begins walking the tree from // the reducer. Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", SelectOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R1", SelectOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getSelProc()); // Matches only GroupByOperators which are reducers, rather than map group by operators, // or multi group by optimization specific operators @@ -109,20 +109,20 @@ private void inferBucketingSorting(List mapRedTasks) throws Semantic // Matches only JoinOperators which are reducers, rather than map joins, SMB map joins, etc. opRules.put(new RuleExactMatch("R3", JoinOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getJoinProc()); - opRules.put(new RuleRegExp("R5", FileSinkOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R5", FileSinkOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getFileSinkProc()); - opRules.put(new RuleRegExp("R7", FilterOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R7", FilterOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getFilterProc()); - opRules.put(new RuleRegExp("R8", LimitOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R8", LimitOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getLimitProc()); - opRules.put(new RuleRegExp("R9", LateralViewForwardOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R9", LateralViewForwardOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getLateralViewForwardProc()); - opRules.put(new RuleRegExp("R10", LateralViewJoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R10", LateralViewJoinOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getLateralViewJoinProc()); // Matches only ForwardOperators which are preceded by some other operator in the tree, // in particular it can't be a reducer (and hence cannot be one of the ForwardOperators // added by the multi group by optimization) - opRules.put(new RuleRegExp("R11", ".+" + ForwardOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R11", ".+" + ForwardOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getForwardProc()); // Matches only ForwardOperators which are reducers and are followed by GroupByOperators // (specific to the multi group by optimization) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java index 6bdb0a7..c8b51dc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CrossProductCheck.java @@ -49,7 +49,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.lib.TaskGraphWalker; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.BaseWork; @@ -253,7 +253,7 @@ private void checkForCrossProduct(String taskName, List analyze(BaseWork work) throws SemanticException { Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", MapJoinOperator.getOperatorName() + opRules.put(new RuleExp("R1", MapJoinOperator.getOperatorName() + "%"), this); Dispatcher disp = new DefaultRuleDispatcher(new NoopProcessor(), opRules, this); GraphWalker ogw = new DefaultGraphWalker(disp); @@ -329,7 +329,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Map analyze(BaseWork work) throws SemanticException { Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", ReduceSinkOperator.getOperatorName() + opRules.put(new RuleExp("R1", ReduceSinkOperator.getOperatorName() + "%"), this); Dispatcher disp = new DefaultRuleDispatcher(new NoopProcessor(), opRules, this); GraphWalker ogw = new DefaultGraphWalker(disp); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java index c0a72b6..9e257a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java @@ -45,7 +45,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.lib.TaskGraphWalker; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -252,7 +252,7 @@ private LocalMapJoinProcCtx adjustLocalTask(MapredLocalTask task) LocalMapJoinProcCtx localMapJoinProcCtx = new LocalMapJoinProcCtx(task, physicalContext .getParseContext()); Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", MapJoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R1", MapJoinOperator.getOperatorName() + "%"), LocalMapJoinProcFactory.getJoinProc()); // The dispatcher fires the processor corresponding to the closest // matching rule and passes the context along diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java index eb8597d..5c3dca9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java @@ -5,11 +5,9 @@ import java.util.Comparator; import java.util.Iterator; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.LinkedHashMap; import java.util.LinkedHashSet; -import java.util.List; import java.util.Set; import java.util.SortedSet; import java.util.Stack; @@ -19,11 +17,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.StatsTask; import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.tez.DagUtils; import org.apache.hadoop.hive.ql.exec.tez.TezTask; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; @@ -33,7 +28,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.lib.TaskGraphWalker; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -41,8 +36,6 @@ import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MergeJoinWork; import org.apache.hadoop.hive.ql.plan.ReduceWork; -import org.apache.hadoop.hive.ql.plan.TezEdgeProperty; -import org.apache.hadoop.hive.ql.plan.TezEdgeProperty.EdgeType; import org.apache.hadoop.hive.ql.plan.TezWork; /** @@ -120,7 +113,7 @@ private void evaluateOperators(BaseWork w, PhysicalContext pctx) throws Semantic final Set mapJoins = new LinkedHashSet(); Map rules = new HashMap(); - rules.put(new RuleRegExp("Map join memory estimator", + rules.put(new RuleExp("Map join memory estimator", MapJoinOperator.getOperatorName() + "%"), new NodeProcessor() { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanOptimizer.java index 080a0e6..67bf5bb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanOptimizer.java @@ -35,7 +35,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer.WalkerCtx; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -54,7 +54,7 @@ public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException { Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%.*" + + opRules.put(new RuleExp("R1", TableScanOperator.getOperatorName() + "%.*" + FilterOperator.getOperatorName() + "%"), new WhereFalseProcessor()); Dispatcher disp = new NullScanTaskDispatcher(pctx, opRules); GraphWalker ogw = new DefaultGraphWalker(disp); @@ -64,8 +64,8 @@ public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException { opRules.clear(); - opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName()+ "%"),new TSMarker()); - opRules.put(new RuleRegExp("R2", LimitOperator.getOperatorName()+ "%"), new Limit0Processor()); + opRules.put(new RuleExp("R1", TableScanOperator.getOperatorName()+ "%"),new TSMarker()); + opRules.put(new RuleExp("R2", LimitOperator.getOperatorName()+ "%"), new Limit0Processor()); disp = new NullScanTaskDispatcher(pctx, opRules); ogw = new DefaultGraphWalker(disp); topNodes = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java index f48d118..7edd056 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java @@ -35,7 +35,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.MapredWork; @@ -82,7 +82,7 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) physicalContext.getParseContext()); Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", CommonJoinOperator.getOperatorName() + "%"), SkewJoinProcFactory.getJoinProc()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 6e86d69..81a4b9f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -70,7 +70,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.lib.TaskGraphWalker; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; @@ -358,9 +358,9 @@ private void convertMapWork(MapWork mapWork, boolean isTez) throws SemanticExcep } private void addMapWorkRules(Map opRules, NodeProcessor np) { - opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + ".*" + opRules.put(new RuleExp("R1", TableScanOperator.getOperatorName() + ".*" + FileSinkOperator.getOperatorName()), np); - opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + ".*" + opRules.put(new RuleExp("R2", TableScanOperator.getOperatorName() + ".*" + ReduceSinkOperator.getOperatorName()), np); } @@ -493,8 +493,8 @@ private boolean getOnlyStructObjectInspectors(ReduceWork reduceWork) throws Sema } private void addReduceWorkRules(Map opRules, NodeProcessor np) { - opRules.put(new RuleRegExp("R1", GroupByOperator.getOperatorName() + ".*"), np); - opRules.put(new RuleRegExp("R2", SelectOperator.getOperatorName() + ".*"), np); + opRules.put(new RuleExp("R1", GroupByOperator.getOperatorName() + ".*"), np); + opRules.put(new RuleExp("R2", SelectOperator.getOperatorName() + ".*"), np); } private boolean validateReduceWork(ReduceWork reduceWork) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java index ae96def..d39b2ac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java @@ -20,7 +20,6 @@ import java.io.Serializable; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -41,8 +40,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; -import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.IndexUtils; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -135,7 +133,7 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) // We set the pushed predicate from the WHERE clause as the filter expr on // all table scan operators, so we look for table scan operators(TS%) - operatorRules.put(new RuleRegExp("RULEWhere", TableScanOperator.getOperatorName() + "%"), + operatorRules.put(new RuleExp("RULEWhere", TableScanOperator.getOperatorName() + "%"), new IndexWhereProcessor(indexes)); return operatorRules; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java index fd42959..8310936 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkReduceSinkMapJoinProc.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.spark.GenSparkProcContext; import org.apache.hadoop.hive.ql.plan.BaseWork; @@ -89,7 +89,7 @@ private boolean hasGroupBy(Operator mapjoinOp, List> childOps = mapjoinOp.getChildOperators(); Map rules = new LinkedHashMap(); SparkMapJoinFollowedByGroupByProcessor processor = new SparkMapJoinFollowedByGroupByProcessor(); - rules.put(new RuleRegExp("GBY", GroupByOperator.getOperatorName() + "%"), processor); + rules.put(new RuleExp("GBY", GroupByOperator.getOperatorName() + "%"), processor); Dispatcher disp = new DefaultRuleDispatcher(null, rules, context); GraphWalker ogw = new DefaultGraphWalker(disp); ArrayList topNodes = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSkewJoinResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSkewJoinResolver.java index 608a0de..e93ebd2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSkewJoinResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkSkewJoinResolver.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalPlanResolver; import org.apache.hadoop.hive.ql.optimizer.physical.SkewJoinResolver; @@ -83,7 +83,7 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) SparkSkewJoinProcCtx skewJoinProcCtx = new SparkSkewJoinProcCtx(task, physicalContext.getParseContext()); Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", CommonJoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R1", CommonJoinOperator.getOperatorName() + "%"), SparkSkewJoinProcFactory.getJoinProc()); Dispatcher disp = new DefaultRuleDispatcher( SparkSkewJoinProcFactory.getDefaultProc(), opRules, skewJoinProcCtx); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java index 4aeeff2..e789c13 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java @@ -37,7 +37,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -51,19 +51,19 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // create a walker which walks the tree in a DFS manner while maintaining the // operator stack. The dispatcher generates the plan from the operator tree Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("TS", TableScanOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("TS", TableScanOperator.getOperatorName() + "%"), StatsRulesProcFactory.getTableScanRule()); - opRules.put(new RuleRegExp("SEL", SelectOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("SEL", SelectOperator.getOperatorName() + "%"), StatsRulesProcFactory.getSelectRule()); - opRules.put(new RuleRegExp("FIL", FilterOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("FIL", FilterOperator.getOperatorName() + "%"), StatsRulesProcFactory.getFilterRule()); - opRules.put(new RuleRegExp("GBY", GroupByOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("GBY", GroupByOperator.getOperatorName() + "%"), StatsRulesProcFactory.getGroupByRule()); - opRules.put(new RuleRegExp("JOIN", CommonJoinOperator.getOperatorName() + "%|" + opRules.put(new RuleExp("JOIN", CommonJoinOperator.getOperatorName() + "%|" + MapJoinOperator.getOperatorName() + "%"), StatsRulesProcFactory.getJoinRule()); - opRules.put(new RuleRegExp("LIM", LimitOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("LIM", LimitOperator.getOperatorName() + "%"), StatsRulesProcFactory.getLimitRule()); - opRules.put(new RuleRegExp("RS", ReduceSinkOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("RS", ReduceSinkOperator.getOperatorName() + "%"), StatsRulesProcFactory.getReduceSinkRule()); // The dispatcher fires the processor corresponding to the closest matching diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java index 9937343..6902788 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java @@ -35,7 +35,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -69,13 +69,13 @@ public ParseContext transform(ParseContext pCtx) throws SemanticException { // create a walker which walks the tree in a DFS manner while maintaining // the operator stack. Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", ReduceSinkOperator.getOperatorName() + "%.*" + UnionOperator.getOperatorName() + "%"), UnionProcFactory.getMapRedUnion()); - opRules.put(new RuleRegExp("R2", + opRules.put(new RuleExp("R2", UnionOperator.getOperatorName() + "%.*" + UnionOperator.getOperatorName() + "%"), UnionProcFactory.getUnknownUnion()); - opRules.put(new RuleRegExp("R3", + opRules.put(new RuleExp("R3", TableScanOperator.getOperatorName() + "%.*" + UnionOperator.getOperatorName() + "%"), UnionProcFactory.getMapUnion()); @@ -104,7 +104,7 @@ public ParseContext transform(ParseContext pCtx) throws SemanticException { SemanticException(ErrorMsg.HIVE_UNION_REMOVE_OPTIMIZATION_NEEDS_SUBDIRECTORIES.getMsg()); } - opRules.put(new RuleRegExp("R5", UnionOperator.getOperatorName() + "%" + + opRules.put(new RuleExp("R5", UnionOperator.getOperatorName() + "%" + ".*" + FileSinkOperator.getOperatorName() + "%"), UnionProcFactory.getUnionNoProcessFile()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java index eeccc4b..1050547 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java @@ -53,7 +53,7 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1; import org.apache.hadoop.hive.ql.optimizer.GenMROperator; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext; @@ -289,25 +289,25 @@ protected void generateTaskTree(List> rootTasks, Pa // the operator stack. // The dispatcher generates the plan from the operator tree Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp(new String("R1"), + opRules.put(new RuleExp(new String("R1"), TableScanOperator.getOperatorName() + "%"), new GenMRTableScan1()); - opRules.put(new RuleRegExp(new String("R2"), + opRules.put(new RuleExp(new String("R2"), TableScanOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"), new GenMRRedSink1()); - opRules.put(new RuleRegExp(new String("R3"), + opRules.put(new RuleExp(new String("R3"), ReduceSinkOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"), new GenMRRedSink2()); - opRules.put(new RuleRegExp(new String("R4"), + opRules.put(new RuleExp(new String("R4"), FileSinkOperator.getOperatorName() + "%"), new GenMRFileSink1()); - opRules.put(new RuleRegExp(new String("R5"), + opRules.put(new RuleExp(new String("R5"), UnionOperator.getOperatorName() + "%"), new GenMRUnion1()); - opRules.put(new RuleRegExp(new String("R6"), + opRules.put(new RuleExp(new String("R6"), UnionOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"), new GenMRRedSink3()); - opRules.put(new RuleRegExp(new String("R7"), + opRules.put(new RuleExp(new String("R7"), MapJoinOperator.getOperatorName() + "%"), MapJoinFactory.getTableScanMapJoin()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java index cc0a7d1..1727f46 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -73,11 +73,11 @@ public TableAccessInfo analyzeTableAccess() throws SemanticException { // Set up the rules for the graph walker for group by and join operators Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", GroupByOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R1", GroupByOperator.getOperatorName() + "%"), new GroupByProcessor(pGraphContext)); - opRules.put(new RuleRegExp("R2", JoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R2", JoinOperator.getOperatorName() + "%"), new JoinProcessor(pGraphContext)); - opRules.put(new RuleRegExp("R3", MapJoinOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("R3", MapJoinOperator.getOperatorName() + "%"), new JoinProcessor(pGraphContext)); TableAccessCtx tableAccessCtx = new TableAccessCtx(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 8ab7cd4..cc178b6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -59,7 +59,7 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption; @@ -259,15 +259,15 @@ private void runStatsDependentOptimizations(OptimizeTezProcContext procCtx, // create a walker which walks the tree in a DFS manner while maintaining // the operator stack. Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("Set parallelism - ReduceSink", + opRules.put(new RuleExp("Set parallelism - ReduceSink", ReduceSinkOperator.getOperatorName() + "%"), new SetReducerParallelism()); - opRules.put(new RuleRegExp("Convert Join to Map-join", + opRules.put(new RuleExp("Convert Join to Map-join", JoinOperator.getOperatorName() + "%"), new ConvertJoinMapJoin()); opRules.put( - new RuleRegExp("Remove dynamic pruning by size", + new RuleExp("Remove dynamic pruning by size", AppMasterEventOperator.getOperatorName() + "%"), new RemoveDynamicPruningBySize()); @@ -293,7 +293,7 @@ private void runDynamicPartitionPruning(OptimizeTezProcContext procCtx, Set opRules = new LinkedHashMap(); opRules.put( - new RuleRegExp(new String("Dynamic Partition Pruning"), FilterOperator.getOperatorName() + new RuleExp(new String("Dynamic Partition Pruning"), FilterOperator.getOperatorName() + "%"), new DynamicPartitionPruningOptimization()); // The dispatcher fires the processor corresponding to the closest matching @@ -330,33 +330,33 @@ protected void generateTaskTree(List> rootTasks, Pa // the operator stack. // The dispatcher generates the plan from the operator tree Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("Split Work - ReduceSink", + opRules.put(new RuleExp("Split Work - ReduceSink", ReduceSinkOperator.getOperatorName() + "%"), genTezWork); - opRules.put(new RuleRegExp("No more walking on ReduceSink-MapJoin", + opRules.put(new RuleExp("No more walking on ReduceSink-MapJoin", MapJoinOperator.getOperatorName() + "%"), new ReduceSinkMapJoinProc()); - opRules.put(new RuleRegExp("Recoginze a Sorted Merge Join operator to setup the right edge and" + opRules.put(new RuleExp("Recoginze a Sorted Merge Join operator to setup the right edge and" + " stop traversing the DummyStore-MapJoin", CommonMergeJoinOperator.getOperatorName() + "%"), new MergeJoinProc()); - opRules.put(new RuleRegExp("Split Work + Move/Merge - FileSink", + opRules.put(new RuleExp("Split Work + Move/Merge - FileSink", FileSinkOperator.getOperatorName() + "%"), new CompositeProcessor(new FileSinkProcessor(), genTezWork)); - opRules.put(new RuleRegExp("Split work - DummyStore", DummyStoreOperator.getOperatorName() + opRules.put(new RuleExp("Split work - DummyStore", DummyStoreOperator.getOperatorName() + "%"), genTezWork); - opRules.put(new RuleRegExp("Handle Potential Analyze Command", + opRules.put(new RuleExp("Handle Potential Analyze Command", TableScanOperator.getOperatorName() + "%"), new ProcessAnalyzeTable(GenTezUtils.getUtils())); - opRules.put(new RuleRegExp("Remember union", + opRules.put(new RuleExp("Remember union", UnionOperator.getOperatorName() + "%"), new UnionProcessor()); - opRules.put(new RuleRegExp("AppMasterEventOperator", + opRules.put(new RuleExp("AppMasterEventOperator", AppMasterEventOperator.getOperatorName() + "%"), new AppMasterEventProcessor()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 0e97530..31e8674 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -52,7 +52,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -157,16 +157,16 @@ public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) // generates the plan from the operator tree Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", HiveParser.TOK_NULL + "%"), + opRules.put(new RuleExp("R1", HiveParser.TOK_NULL + "%"), tf.getNullExprProcessor()); - opRules.put(new RuleRegExp("R2", HiveParser.Number + "%|" + + opRules.put(new RuleExp("R2", HiveParser.Number + "%|" + HiveParser.TinyintLiteral + "%|" + HiveParser.SmallintLiteral + "%|" + HiveParser.BigintLiteral + "%|" + HiveParser.DecimalLiteral + "%"), tf.getNumExprProcessor()); opRules - .put(new RuleRegExp("R3", HiveParser.Identifier + "%|" + .put(new RuleExp("R3", HiveParser.Identifier + "%|" + HiveParser.StringLiteral + "%|" + HiveParser.TOK_CHARSETLITERAL + "%|" + HiveParser.TOK_STRINGLITERALSEQUENCE + "%|" + "%|" + HiveParser.KW_IF + "%|" + HiveParser.KW_CASE + "%|" @@ -175,11 +175,11 @@ public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) + HiveParser.KW_STRUCT + "%|" + HiveParser.KW_EXISTS + "%|" + HiveParser.TOK_SUBQUERY_OP_NOTIN + "%"), tf.getStrExprProcessor()); - opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|" + opRules.put(new RuleExp("R4", HiveParser.KW_TRUE + "%|" + HiveParser.KW_FALSE + "%"), tf.getBoolExprProcessor()); - opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%|" + opRules.put(new RuleExp("R5", HiveParser.TOK_DATELITERAL + "%|" + HiveParser.TOK_TIMESTAMPLITERAL + "%"), tf.getDateTimeExprProcessor()); - opRules.put(new RuleRegExp("R6", + opRules.put(new RuleExp("R6", HiveParser.TOK_INTERVAL_YEAR_MONTH_LITERAL + "%|" + HiveParser.TOK_INTERVAL_DAY_TIME_LITERAL + "%|" + HiveParser.TOK_INTERVAL_YEAR_LITERAL + "%|" @@ -188,9 +188,9 @@ public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) + HiveParser.TOK_INTERVAL_HOUR_LITERAL + "%|" + HiveParser.TOK_INTERVAL_MINUTE_LITERAL + "%|" + HiveParser.TOK_INTERVAL_SECOND_LITERAL + "%"), tf.getIntervalExprProcessor()); - opRules.put(new RuleRegExp("R7", HiveParser.TOK_TABLE_OR_COL + "%"), + opRules.put(new RuleExp("R7", HiveParser.TOK_TABLE_OR_COL + "%"), tf.getColumnExprProcessor()); - opRules.put(new RuleRegExp("R8", HiveParser.TOK_SUBQUERY_OP + "%"), + opRules.put(new RuleExp("R8", HiveParser.TOK_SUBQUERY_OP + "%"), tf.getSubQueryExprProcessor()); // The dispatcher fires the processor corresponding to the closest matching diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java index 19aae70..adf6b78 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java @@ -55,7 +55,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.lib.TypeRule; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -109,7 +109,7 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, // create a walker which walks the tree in a DFS manner while maintaining // the operator stack. Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("Set parallelism - ReduceSink", + opRules.put(new RuleExp("Set parallelism - ReduceSink", ReduceSinkOperator.getOperatorName() + "%"), new SetSparkReducerParallelism()); @@ -148,20 +148,20 @@ protected void generateTaskTree(List> rootTasks, Pa // create a walker which walks the tree in a DFS manner while maintaining // the operator stack. The dispatcher generates the plan from the operator tree Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("Split Work - ReduceSink", + opRules.put(new RuleExp("Split Work - ReduceSink", ReduceSinkOperator.getOperatorName() + "%"), genSparkWork); opRules.put(new TypeRule(MapJoinOperator.class), new SparkReduceSinkMapJoinProc()); - opRules.put(new RuleRegExp("Split Work + Move/Merge - FileSink", + opRules.put(new RuleExp("Split Work + Move/Merge - FileSink", FileSinkOperator.getOperatorName() + "%"), new CompositeProcessor(new SparkFileSinkProcessor(), genSparkWork)); - opRules.put(new RuleRegExp("Handle Analyze Command", + opRules.put(new RuleExp("Handle Analyze Command", TableScanOperator.getOperatorName() + "%"), new SparkProcessAnalyzeTable(GenSparkUtils.getUtils())); - opRules.put(new RuleRegExp("Remember union", UnionOperator.getOperatorName() + "%"), + opRules.put(new RuleExp("Remember union", UnionOperator.getOperatorName() + "%"), new NodeProcessor() { @Override public Object process(Node n, Stack s, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java index 3a07b17..785271a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -268,12 +269,12 @@ public static ExprWalkerInfo extractPushdownPreds(OpWalkerInfo opContext, // generates the plan from the operator tree Map exprRules = new LinkedHashMap(); exprRules.put( - new RuleRegExp("R1", ExprNodeColumnDesc.class.getName() + "%"), + new RuleExp("R1", ExprNodeColumnDesc.class.getName() + "%"), getColumnProcessor()); exprRules.put( - new RuleRegExp("R2", ExprNodeFieldDesc.class.getName() + "%"), + new RuleExp("R2", ExprNodeFieldDesc.class.getName() + "%"), getFieldProcessor()); - exprRules.put(new RuleRegExp("R3", ExprNodeGenericFuncDesc.class.getName() + exprRules.put(new RuleExp("R3", ExprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor()); // The dispatcher fires the processor corresponding to the closest matching diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java index 7f26f0f..18083e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -92,34 +93,34 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { OpWalkerInfo opWalkerInfo = new OpWalkerInfo(pGraphContext); Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", + opRules.put(new RuleExp("R1", FilterOperator.getOperatorName() + "%"), OpProcFactory.getFilterProc()); - opRules.put(new RuleRegExp("R2", + opRules.put(new RuleExp("R2", PTFOperator.getOperatorName() + "%"), OpProcFactory.getPTFProc()); - opRules.put(new RuleRegExp("R3", + opRules.put(new RuleExp("R3", CommonJoinOperator.getOperatorName() + "%"), OpProcFactory.getJoinProc()); - opRules.put(new RuleRegExp("R4", + opRules.put(new RuleExp("R4", TableScanOperator.getOperatorName() + "%"), OpProcFactory.getTSProc()); - opRules.put(new RuleRegExp("R5", + opRules.put(new RuleExp("R5", ScriptOperator.getOperatorName() + "%"), OpProcFactory.getSCRProc()); - opRules.put(new RuleRegExp("R6", + opRules.put(new RuleExp("R6", LimitOperator.getOperatorName() + "%"), OpProcFactory.getLIMProc()); - opRules.put(new RuleRegExp("R7", + opRules.put(new RuleExp("R7", UDTFOperator.getOperatorName() + "%"), OpProcFactory.getUDTFProc()); - opRules.put(new RuleRegExp("R8", + opRules.put(new RuleExp("R8", LateralViewForwardOperator.getOperatorName() + "%"), OpProcFactory.getLVFProc()); - opRules.put(new RuleRegExp("R9", + opRules.put(new RuleExp("R9", LateralViewJoinOperator.getOperatorName() + "%"), OpProcFactory.getLVJProc()); - opRules.put(new RuleRegExp("R10", + opRules.put(new RuleExp("R10", ReduceSinkOperator.getOperatorName() + "%"), OpProcFactory.getRSProc()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java index ea1f713..837ab54 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -65,7 +65,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { pGraphContext = pctx; Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", "(" + + opRules.put(new RuleExp("R1", "(" + FilterOperator.getOperatorName() + "%" + ReduceSinkOperator.getOperatorName() + "%" + JoinOperator.getOperatorName() + "%)"), new JoinTransitive()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java index 363e49e..7cc2fa1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java @@ -45,7 +45,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.lib.RuleExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -73,7 +73,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { } Map opRules = new LinkedHashMap(); - opRules.put(new RuleRegExp("R1", "(" + + opRules.put(new RuleExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + ".*" + ReduceSinkOperator.getOperatorName() + "%" + JoinOperator.getOperatorName() + "%)"), new JoinSynthetic());