Index: ql/src/test/org/apache/hadoop/hive/ql/tool/TestLineageInfo.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/tool/TestLineageInfo.java (revision 726865) +++ ql/src/test/org/apache/hadoop/hive/ql/tool/TestLineageInfo.java (working copy) @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.tool; import java.util.TreeSet; -import java.util.Vector; import org.apache.hadoop.hive.ql.tools.LineageInfo; @@ -27,6 +26,22 @@ public class TestLineageInfo extends TestCase { + /** + * Checks whether the test outputs match the expected outputs + * @param lep The LineageInfo extracted from the test + * @param i The set of input tables + * @param o The set of output tables + */ + private void checkOutput(LineageInfo lep, TreeSet i, TreeSet o) { + + if ( !i.equals(lep.getInputTableList())){ + fail("Input table not same"); + } + if (! o.equals(lep.getOutputTableList())){ + fail("Output table not same"); + } + } + public void testSimpleQuery(){ LineageInfo lep = new LineageInfo(); try{ @@ -36,19 +51,12 @@ TreeSet o = new TreeSet(); i.add("srcpart"); o.add("dest1"); - if ( !i.equals(lep.getInputTableList())){ - fail("Input table not same"); - } - if (! o.equals(lep.getOutputTableList())){ - fail("Output table not same"); - } - + checkOutput(lep, i, o); } catch (Exception e) { e.printStackTrace(); fail("Failed"); } - } public void testSimpleQuery2(){ @@ -59,21 +67,13 @@ ); TreeSet i = new TreeSet(); TreeSet o = new TreeSet(); - i.add("src"); - - if ( !i.equals(lep.getInputTableList())){ - fail("Input table not same"); - } - if (! o.equals(lep.getOutputTableList())){ - fail("Output table not same"); - } - + i.add("src"); + checkOutput(lep, i, o); } catch (Exception e) { e.printStackTrace(); fail("Failed"); - } - + } } public void testSimpleQuery3(){ @@ -84,21 +84,14 @@ ); TreeSet i = new TreeSet(); TreeSet o = new TreeSet(); - i.add("src"); - i.add("src1"); - if ( !i.equals(lep.getInputTableList())){ - fail("Input table not same"); - } - if (! o.equals(lep.getOutputTableList())){ - fail("Output table not same"); - } - + i.add("src"); + i.add("src1"); + checkOutput(lep, i, o); } catch (Exception e) { e.printStackTrace(); fail("Failed"); } - } public void testSimpleQuery4(){ @@ -108,14 +101,9 @@ "FROM ( FROM ( FROM src1 src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20) a RIGHT OUTER JOIN ( FROM src2 src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25) b ON (a.c1 = b.c3) SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4) c SELECT c.c1, c.c2, c.c3, c.c4" ); TreeSet i = new TreeSet(); TreeSet o = new TreeSet(); - i.add("src1"); - i.add("src2"); - if ( !i.equals(lep.getInputTableList())){ - fail("Input table not same"); - } - if (! o.equals(lep.getOutputTableList())){ - fail("Output table not same"); - } + i.add("src1"); + i.add("src2"); + checkOutput(lep, i, o); } catch (Exception e) { e.printStackTrace(); Index: ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (revision 726865) +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (working copy) @@ -35,7 +35,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.antlr.runtime.tree.CommonTree; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -48,6 +47,7 @@ import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.ParseDriver; import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; @@ -463,7 +463,7 @@ return exitVal; } - public int checkParseResults(String tname, CommonTree tree) throws Exception { + public int checkParseResults(String tname, ASTNode tree) throws Exception { if (tree != null) { File parseDir = new File(outDir, "parse"); @@ -663,18 +663,18 @@ return exitVal; } - public CommonTree parseQuery(String tname) throws Exception { + public ASTNode parseQuery(String tname) throws Exception { return pd.parse(qMap.get(tname)); } - public List> analyzeAST(CommonTree ast) throws Exception { + public List> analyzeAST(ASTNode ast) throws Exception { // Do semantic analysis and plan generation Context ctx = new Context(conf); ctx.makeScratchDir(); while((ast.getToken() == null) && (ast.getChildCount() > 0)) { - ast = (CommonTree)ast.getChild(0); + ast = (ASTNode)ast.getChild(0); } sem.analyze(ast, ctx); Index: ql/src/test/templates/TestParse.vm =================================================================== --- ql/src/test/templates/TestParse.vm (revision 726865) +++ ql/src/test/templates/TestParse.vm (working copy) @@ -10,9 +10,6 @@ import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.exec.Task; -import org.antlr.runtime.*; -import org.antlr.runtime.tree.*; - public class $className extends TestCase { private QTestUtil qt; @@ -58,7 +55,7 @@ try { System.out.println("Begin query: " + "$fname"); qt.init("$fname"); - CommonTree tree = qt.parseQuery("$fname"); + ASTNode tree = qt.parseQuery("$fname"); int ecode = qt.checkParseResults("$fname", tree); if (ecode != 0) { fail("Parse has unexpected out with error code = " + ecode); Index: ql/src/test/templates/TestParseNegative.vm =================================================================== --- ql/src/test/templates/TestParseNegative.vm (revision 726865) +++ ql/src/test/templates/TestParseNegative.vm (working copy) @@ -10,9 +10,6 @@ import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.exec.Task; -import org.antlr.runtime.*; -import org.antlr.runtime.tree.*; - public class $className extends TestCase { private QTestUtil qt; @@ -58,7 +55,7 @@ try { System.out.println("Begin query: " + "$fname"); qt.init("$fname"); - CommonTree tree = qt.parseQuery("$fname"); + ASTNode tree = qt.parseQuery("$fname"); List> tasks = qt.analyzeAST(tree); fail("Unexpected success for query: " + "$fname"); } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorProcessorContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorProcessorContext.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorProcessorContext.java (working copy) @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.optimizer; - -/** - * Operator Processor Context - */ -public abstract class OperatorProcessorContext { -} Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java (working copy) @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.plan.partitionDesc; import org.apache.hadoop.hive.ql.plan.fileSinkDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.metadata.*; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -54,7 +55,7 @@ * Processor Context for creating map reduce task. Walk the tree in a DFS manner and process the nodes. Some state is * maintained about the current nodes visited so far. */ -public class GenMRProcContext extends OperatorProcessorContext { +public class GenMRProcContext extends NodeProcessorCtx { /** * GenMapRedCtx is used to keep track of the current state. Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java (working copy) @@ -23,26 +23,28 @@ import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.OperatorProcessor; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; /** * Processor for the rule - no specific rule fired */ -public class GenMROperator implements OperatorProcessor { +public class GenMROperator implements NodeProcessor { public GenMROperator() { } /** * Reduce Scan encountered - * @param op the reduce sink operator encountered - * @param opProcCtx context + * @param nd the reduce sink operator encountered + * @param procCtx context */ - public void process(Operator op, OperatorProcessorContext opProcCtx) throws SemanticException { - GenMRProcContext ctx = (GenMRProcContext)opProcCtx; + public void process(Node nd, NodeProcessorCtx procCtx) throws SemanticException { + GenMRProcContext ctx = (GenMRProcContext)procCtx; Map, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx(); - mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId())); + mapCurrCtx.put((Operator)nd, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId())); } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (revision 0) @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; +import org.apache.hadoop.hive.ql.exec.FilterOperator; +import org.apache.hadoop.hive.ql.exec.GroupByOperator; +import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.exec.ScriptOperator; +import org.apache.hadoop.hive.ql.exec.SelectOperator; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.parse.OpParseContext; +import org.apache.hadoop.hive.ql.parse.RowResolver; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.aggregationDesc; +import org.apache.hadoop.hive.ql.plan.exprNodeDesc; +import org.apache.hadoop.hive.ql.plan.groupByDesc; +import org.apache.hadoop.hive.ql.plan.reduceSinkDesc; +import org.apache.hadoop.hive.ql.plan.selectDesc; + +/** + * Factory for generating the different node processors used by ColumnPruner. + */ +public class ColumnPrunerProcFactory { + + /** + * Node Processor for Column Pruning on Filter Operators. + */ + public static class ColumnPrunerFilterProc implements NodeProcessor { + public void process(Node nd, NodeProcessorCtx ctx) throws SemanticException { + FilterOperator op = (FilterOperator)nd; + ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx)ctx; + exprNodeDesc condn = op.getConf().getPredicate(); + // get list of columns used in the filter + List cl = condn.getCols(); + // merge it with the downstream col list + cppCtx.getPrunedColLists().put(op, Utilities.mergeUniqElems(cppCtx.genColLists(op), cl)); + } + } + + /** + * Factory method to get the ColumnPrunerFilterProc class. + * @return ColumnPrunerFilterProc + */ + public static ColumnPrunerFilterProc getFilterProc() { + return new ColumnPrunerFilterProc(); + } + + /** + * Node Processor for Column Pruning on Group By Operators. + */ + public static class ColumnPrunerGroupByProc implements NodeProcessor { + public void process(Node nd, NodeProcessorCtx ctx) throws SemanticException { + GroupByOperator op = (GroupByOperator)nd; + ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx)ctx; + List colLists = new ArrayList(); + groupByDesc conf = op.getConf(); + ArrayList keys = conf.getKeys(); + for (exprNodeDesc key : keys) + colLists = Utilities.mergeUniqElems(colLists, key.getCols()); + + ArrayList aggrs = conf.getAggregators(); + for (aggregationDesc aggr : aggrs) { + ArrayList params = aggr.getParameters(); + for (exprNodeDesc param : params) + colLists = Utilities.mergeUniqElems(colLists, param.getCols()); + } + + cppCtx.getPrunedColLists().put(op, colLists); + } + } + + /** + * Factory method to get the ColumnPrunerGroupByProc class. + * @return ColumnPrunerGroupByProc + */ + public static ColumnPrunerGroupByProc getGroupByProc() { + return new ColumnPrunerGroupByProc(); + } + + /** + * The Default Node Processor for Column Pruning. + */ + public static class ColumnPrunerDefaultProc implements NodeProcessor { + public void process(Node nd, NodeProcessorCtx ctx) throws SemanticException { + ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx)ctx; + cppCtx.getPrunedColLists().put((Operator)nd, + cppCtx.genColLists((Operator)nd)); + } + } + + /** + * Factory method to get the ColumnPrunerDefaultProc class. + * @return ColumnPrunerDefaultProc + */ + public static ColumnPrunerDefaultProc getDefaultProc() { + return new ColumnPrunerDefaultProc(); + } + + /** + * The Node Processor for Column Pruning on Reduce Sink Operators. + */ + public static class ColumnPrunerReduceSinkProc implements NodeProcessor { + public void process(Node nd, NodeProcessorCtx ctx) throws SemanticException { + ReduceSinkOperator op = (ReduceSinkOperator)nd; + ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx)ctx; + HashMap, OpParseContext> opToParseCtxMap = + cppCtx.getOpToParseCtxMap(); + RowResolver redSinkRR = opToParseCtxMap.get(op).getRR(); + reduceSinkDesc conf = op.getConf(); + List> childOperators = op.getChildOperators(); + List> parentOperators = op.getParentOperators(); + List childColLists = new ArrayList(); + + for(Operator child: childOperators) + childColLists = Utilities.mergeUniqElems(childColLists, cppCtx.getPrunedColLists().get(child)); + + List colLists = new ArrayList(); + ArrayList keys = conf.getKeyCols(); + for (exprNodeDesc key : keys) + colLists = Utilities.mergeUniqElems(colLists, key.getCols()); + + if ((childOperators.size() == 1) && (childOperators.get(0) instanceof JoinOperator)) { + assert parentOperators.size() == 1; + Operator par = parentOperators.get(0); + RowResolver parRR = opToParseCtxMap.get(par).getRR(); + RowResolver childRR = opToParseCtxMap.get(childOperators.get(0)).getRR(); + + for (String childCol : childColLists) { + String [] nm = childRR.reverseLookup(childCol); + ColumnInfo cInfo = redSinkRR.get(nm[0],nm[1]); + if (cInfo != null) { + cInfo = parRR.get(nm[0], nm[1]); + if (!colLists.contains(cInfo.getInternalName())) + colLists.add(cInfo.getInternalName()); + } + } + } + else { + // Reduce Sink contains the columns needed - no need to aggregate from children + ArrayList vals = conf.getValueCols(); + for (exprNodeDesc val : vals) + colLists = Utilities.mergeUniqElems(colLists, val.getCols()); + } + + cppCtx.getPrunedColLists().put(op, colLists); + } + } + + /** + * The Factory method to get ColumnPrunerReduceSinkProc class. + * @return ColumnPrunerReduceSinkProc + */ + public static ColumnPrunerReduceSinkProc getReduceSinkProc() { + return new ColumnPrunerReduceSinkProc(); + } + + /** + * The Node Processor for Column Pruning on Select Operators. + */ + public static class ColumnPrunerSelectProc implements NodeProcessor { + public void process(Node nd, NodeProcessorCtx ctx) throws SemanticException { + SelectOperator op = (SelectOperator)nd; + ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx)ctx; + List cols = new ArrayList(); + + if(op.getChildOperators() != null) { + for(Operator child: op.getChildOperators()) { + // If one of my children is a FileSink or Script, return all columns. + // Without this break, a bug in ReduceSink to Extract edge column pruning will manifest + // which should be fixed before remove this + if ((child instanceof FileSinkOperator) || (child instanceof ScriptOperator)) { + cppCtx.getPrunedColLists().put(op, cppCtx.getColsFromSelectExpr(op)); + return; + } + cols = Utilities.mergeUniqElems(cols, cppCtx.getPrunedColLists().get(child)); + } + } + + selectDesc conf = op.getConf(); + if (conf.isSelectStar() && !cols.isEmpty()) { + // The input to the select does not matter. Go over the expressions + // and return the ones which have a marked column + cppCtx.getPrunedColLists().put(op, cppCtx.getSelectColsFromChildren(op, cols)); + return; + } + cppCtx.getPrunedColLists().put(op, cppCtx.getColsFromSelectExpr(op)); + } + } + + /** + * The Factory method to get the ColumnPrunerSelectProc class. + * @return ColumnPrunerSelectProc + */ + public static ColumnPrunerSelectProc getSelectProc() { + return new ColumnPrunerSelectProc(); + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (working copy) @@ -22,12 +22,12 @@ import java.util.Map; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.parse.OperatorProcessor; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; -import org.apache.hadoop.hive.ql.plan.mapredWork; -import org.apache.hadoop.hive.ql.metadata.*; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; @@ -35,16 +35,17 @@ /** * Processor for the rule - table scan */ -public class GenMRTableScan1 implements OperatorProcessor { +public class GenMRTableScan1 implements NodeProcessor { public GenMRTableScan1() { } /** * Table Sink encountered - * @param op the table sink operator encountered + * @param nd the table sink operator encountered * @param opProcCtx context */ - public void process(TableScanOperator op, OperatorProcessorContext opProcCtx) throws SemanticException { + public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException { + TableScanOperator op = (TableScanOperator)nd; GenMRProcContext ctx = (GenMRProcContext)opProcCtx; ParseContext parseCtx = ctx.getParseCtx(); Map, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx(); @@ -67,14 +68,5 @@ assert false; } - /** - * Table Sink encountered - * @param op the table sink operator encountered - * @param opProcCtx context - */ - public void process(Operator op, OperatorProcessorContext opProcCtx) throws SemanticException { - // should never be called - assert false; - } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java (working copy) @@ -22,42 +22,34 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Stack; - import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.ExtractOperator; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; -import org.apache.hadoop.hive.ql.exec.FilterOperator; -import org.apache.hadoop.hive.ql.exec.GroupByOperator; -import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.ScriptOperator; import org.apache.hadoop.hive.ql.exec.SelectOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.parse.DefaultDispatcher; -import org.apache.hadoop.hive.ql.parse.Dispatcher; -import org.apache.hadoop.hive.ql.parse.DefaultOpGraphWalker; -import org.apache.hadoop.hive.ql.parse.OpGraphWalker; +import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; +import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; +import org.apache.hadoop.hive.ql.lib.Dispatcher; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.GraphWalker; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.Rule; +import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.parse.OpParseContext; -import org.apache.hadoop.hive.ql.parse.OperatorProcessor; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.QB; import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.aggregationDesc; import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.exprNodeDesc; -import org.apache.hadoop.hive.ql.plan.groupByDesc; -import org.apache.hadoop.hive.ql.plan.reduceSinkDesc; import org.apache.hadoop.hive.ql.plan.selectDesc; -import org.apache.hadoop.hive.ql.parse.Rule; /** * Implementation of one of the rule-based optimization steps. ColumnPruner gets the current operator tree. The \ @@ -162,16 +154,31 @@ boolean done = true; // generate pruned column list for all relevant operators - ColumnPrunerProcessor cpp = new ColumnPrunerProcessor(opToParseCtxMap); - Dispatcher disp = new DefaultDispatcher(cpp); - OpGraphWalker ogw = new ColumnPrunerWalker(disp); - ogw.startWalking(pGraphContext.getTopOps().values()); + ColumnPrunerProcCtx cppCtx = new ColumnPrunerProcCtx(opToParseCtxMap); + + // create a walker which walks the tree in a DFS manner while maintaining the operator stack. The dispatcher + // generates the plan from the operator tree + Map opRules = new LinkedHashMap(); + opRules.put(new RuleRegExp(new String("R1"), "FIL"), ColumnPrunerProcFactory.getFilterProc()); + opRules.put(new RuleRegExp(new String("R2"), "GBY"), ColumnPrunerProcFactory.getGroupByProc()); + opRules.put(new RuleRegExp(new String("R3"), "RS"), ColumnPrunerProcFactory.getReduceSinkProc()); + opRules.put(new RuleRegExp(new String("R4"), "SEL"), ColumnPrunerProcFactory.getSelectProc()); + // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along + Dispatcher disp = new DefaultRuleDispatcher(ColumnPrunerProcFactory.getDefaultProc(), opRules, cppCtx); + GraphWalker ogw = new ColumnPrunerWalker(disp); + + // Create a list of topop nodes + ArrayList topNodes = new ArrayList(); + topNodes.addAll(pGraphContext.getTopOps().values()); + ogw.startWalking(topNodes); + // create a new select operator if any of input tables' columns can be pruned for (String alias_id : pGraphContext.getTopOps().keySet()) { Operator topOp = pGraphContext.getTopOps().get(alias_id); - List colNames = cpp.getPrunedColList(topOp); + List colNames = cppCtx.getPrunedColList(topOp); + // do we need to push a SELECT - all the columns of the table are not used if (pushSelect(topOp, colNames)) { topOp.setChildOperators(null); @@ -198,157 +205,11 @@ } return pGraphContext; } - + /** - * Column pruner processor - **/ - public static class ColumnPrunerProcessor implements OperatorProcessor { - private Map,List> prunedColLists = - new HashMap, List>(); - private HashMap, OpParseContext> opToParseCtxMap; - - public ColumnPrunerProcessor(HashMap, OpParseContext> opToParseContextMap) { - this.opToParseCtxMap = opToParseContextMap; - } - - /** - * @return the prunedColLists - */ - public List getPrunedColList(Operator op) { - return prunedColLists.get(op); - } - - private List genColLists(Operator curOp) throws SemanticException { - List colList = new ArrayList(); - if(curOp.getChildOperators() != null) { - for(Operator child: curOp.getChildOperators()) - colList = Utilities.mergeUniqElems(colList, prunedColLists.get(child)); - } - return colList; - } - - public void process(FilterOperator op, OperatorProcessorContext ctx) throws SemanticException { - exprNodeDesc condn = op.getConf().getPredicate(); - // get list of columns used in the filter - List cl = condn.getCols(); - // merge it with the downstream col list - prunedColLists.put(op, Utilities.mergeUniqElems(genColLists(op), cl)); - } - - public void process(GroupByOperator op, OperatorProcessorContext ctx) throws SemanticException { - List colLists = new ArrayList(); - groupByDesc conf = op.getConf(); - ArrayList keys = conf.getKeys(); - for (exprNodeDesc key : keys) - colLists = Utilities.mergeUniqElems(colLists, key.getCols()); - - ArrayList aggrs = conf.getAggregators(); - for (aggregationDesc aggr : aggrs) { - ArrayList params = aggr.getParameters(); - for (exprNodeDesc param : params) - colLists = Utilities.mergeUniqElems(colLists, param.getCols()); - } - - prunedColLists.put(op, colLists); - } - - public void process(Operator op, OperatorProcessorContext ctx) throws SemanticException { - prunedColLists.put(op, genColLists(op)); - } - - public void process(ReduceSinkOperator op, OperatorProcessorContext ctx) throws SemanticException { - RowResolver redSinkRR = opToParseCtxMap.get(op).getRR(); - reduceSinkDesc conf = op.getConf(); - List> childOperators = op.getChildOperators(); - List> parentOperators = op.getParentOperators(); - List childColLists = new ArrayList(); - - for(Operator child: childOperators) - childColLists = Utilities.mergeUniqElems(childColLists, prunedColLists.get(child)); - - List colLists = new ArrayList(); - ArrayList keys = conf.getKeyCols(); - for (exprNodeDesc key : keys) - colLists = Utilities.mergeUniqElems(colLists, key.getCols()); - - if ((childOperators.size() == 1) && (childOperators.get(0) instanceof JoinOperator)) { - assert parentOperators.size() == 1; - Operator par = parentOperators.get(0); - RowResolver parRR = opToParseCtxMap.get(par).getRR(); - RowResolver childRR = opToParseCtxMap.get(childOperators.get(0)).getRR(); - - for (String childCol : childColLists) { - String [] nm = childRR.reverseLookup(childCol); - ColumnInfo cInfo = redSinkRR.get(nm[0],nm[1]); - if (cInfo != null) { - cInfo = parRR.get(nm[0], nm[1]); - if (!colLists.contains(cInfo.getInternalName())) - colLists.add(cInfo.getInternalName()); - } - } - } - else { - // Reduce Sink contains the columns needed - no need to aggregate from children - ArrayList vals = conf.getValueCols(); - for (exprNodeDesc val : vals) - colLists = Utilities.mergeUniqElems(colLists, val.getCols()); - } - - prunedColLists.put(op, colLists); - } - - public void process(SelectOperator op, OperatorProcessorContext ctx) throws SemanticException { - List cols = new ArrayList(); - - if(op.getChildOperators() != null) { - for(Operator child: op.getChildOperators()) { - // If one of my children is a FileSink or Script, return all columns. - // Without this break, a bug in ReduceSink to Extract edge column pruning will manifest - // which should be fixed before remove this - if ((child instanceof FileSinkOperator) || (child instanceof ScriptOperator)) { - prunedColLists.put(op, getColsFromSelectExpr(op)); - return; - } - cols = Utilities.mergeUniqElems(cols, prunedColLists.get(child)); - } - } - - selectDesc conf = op.getConf(); - if (conf.isSelectStar() && !cols.isEmpty()) { - // The input to the select does not matter. Go over the expressions - // and return the ones which have a marked column - prunedColLists.put(op, getSelectColsFromChildren(op, cols)); - return; - } - prunedColLists.put(op, getColsFromSelectExpr(op)); - } - - private List getColsFromSelectExpr(SelectOperator op) { - List cols = new ArrayList(); - selectDesc conf = op.getConf(); - ArrayList exprList = conf.getColList(); - for (exprNodeDesc expr : exprList) - cols = Utilities.mergeUniqElems(cols, expr.getCols()); - return cols; - } - - private List getSelectColsFromChildren(SelectOperator op, List colList) { - List cols = new ArrayList(); - selectDesc conf = op.getConf(); - ArrayList selectExprs = conf.getColList(); - - for (String col : colList) { - // col is the internal name i.e. position within the expression list - exprNodeDesc expr = selectExprs.get(Integer.parseInt(col)); - cols = Utilities.mergeUniqElems(cols, expr.getCols()); - } - return cols; - } - } - /** * Walks the op tree in post order fashion (skips selects with file sink or script op children) */ - public static class ColumnPrunerWalker extends DefaultOpGraphWalker { + public static class ColumnPrunerWalker extends DefaultGraphWalker { public ColumnPrunerWalker(Dispatcher disp) { super(disp); @@ -358,30 +219,33 @@ * Walk the given operator */ @Override - public void walk(Operator op) throws SemanticException { + public void walk(Node nd) throws SemanticException { boolean walkChildren = true; + opStack.push(nd); // no need to go further down for a select op with a file sink or script child // since all cols are needed for these ops - if(op instanceof SelectOperator) { - for(Operator child: op.getChildOperators()) { + if(nd instanceof SelectOperator) { + for(Node child: nd.getChildren()) { if ((child instanceof FileSinkOperator) || (child instanceof ScriptOperator)) walkChildren = false; } } - if((op.getChildOperators() == null) - || getDispatchedList().containsAll(op.getChildOperators()) + if((nd.getChildren() == null) + || getDispatchedList().containsAll(nd.getChildren()) || !walkChildren) { // all children are done or no need to walk the children - dispatch(op, null); + dispatch(nd, opStack); + opStack.pop(); return; } // move all the children to the front of queue - getToWalk().removeAll(op.getChildOperators()); - getToWalk().addAll(0, op.getChildOperators()); + getToWalk().removeAll(nd.getChildren()); + getToWalk().addAll(0, nd.getChildren()); // add self to the end of the queue - getToWalk().add(op); + getToWalk().add(nd); + opStack.pop(); } } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java (working copy) @@ -18,54 +18,35 @@ package org.apache.hadoop.hive.ql.optimizer; -import java.util.List; -import java.util.ArrayList; import java.util.Map; import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Set; -import java.util.Stack; import java.io.Serializable; -import java.io.File; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.FileSinkOperator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.TaskFactory; -import org.apache.hadoop.hive.ql.exec.OperatorFactory; import org.apache.hadoop.hive.ql.plan.mapredWork; -import org.apache.hadoop.hive.ql.plan.reduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.tableDesc; -import org.apache.hadoop.hive.ql.plan.partitionDesc; -import org.apache.hadoop.hive.ql.plan.fileSinkDesc; -import org.apache.hadoop.hive.ql.plan.PlanUtils; -import org.apache.hadoop.hive.ql.metadata.*; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.parse.OperatorProcessor; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; /** * Processor for the rule - table scan followed by reduce sink */ -public class GenMRRedSink1 implements OperatorProcessor { +public class GenMRRedSink1 implements NodeProcessor { public GenMRRedSink1() { } /** * Reduce Scan encountered - * @param op the reduce sink operator encountered + * @param nd the reduce sink operator encountered * @param opProcCtx context */ - public void process(ReduceSinkOperator op, OperatorProcessorContext opProcCtx) throws SemanticException { + public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException { + ReduceSinkOperator op = (ReduceSinkOperator)nd; GenMRProcContext ctx = (GenMRProcContext)opProcCtx; Map, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx(); @@ -100,13 +81,4 @@ mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId())); } - /** - * Reduce Scan encountered - * @param op the reduce sink operator encountered - * @param opProcCtx context - */ - public void process(Operator op, OperatorProcessorContext opProcCtx) throws SemanticException { - // should never be called - assert false; - } } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java (working copy) @@ -23,32 +23,28 @@ import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.TaskFactory; -import org.apache.hadoop.hive.ql.exec.OperatorFactory; -import org.apache.hadoop.hive.ql.plan.mapredWork; -import org.apache.hadoop.hive.ql.plan.reduceSinkDesc; -import org.apache.hadoop.hive.ql.metadata.*; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.parse.OperatorProcessor; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; /** * Processor for the rule - reduce sink followed by reduce sink */ -public class GenMRRedSink2 implements OperatorProcessor { +public class GenMRRedSink2 implements NodeProcessor { public GenMRRedSink2() { } /** * Reduce Scan encountered - * @param op the reduce sink operator encountered + * @param nd the reduce sink operator encountered * @param opProcCtx context */ - public void process(ReduceSinkOperator op, OperatorProcessorContext opProcCtx) throws SemanticException { + public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException { + ReduceSinkOperator op = (ReduceSinkOperator)nd; GenMRProcContext ctx = (GenMRProcContext)opProcCtx; Map, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx(); @@ -75,14 +71,4 @@ mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId())); } - /** - * Reduce Scan encountered - * @param op the operator encountered - * @param opProcCtx context - */ - public void process(Operator op, OperatorProcessorContext opProcCtx) throws SemanticException { - // should never be called - assert false; - } - } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java (revision 0) @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.optimizer; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.SelectOperator; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.parse.OpParseContext; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.exprNodeDesc; +import org.apache.hadoop.hive.ql.plan.selectDesc; + +/** + * This class implements the processor context for Column Pruner. + */ +public class ColumnPrunerProcCtx extends NodeProcessorCtx { + + private Map,List> prunedColLists; + + private HashMap, OpParseContext> opToParseCtxMap; + + public ColumnPrunerProcCtx(HashMap, OpParseContext> opToParseContextMap) { + prunedColLists = new HashMap, List>(); + this.opToParseCtxMap = opToParseContextMap; + } + + /** + * @return the prunedColLists + */ + public List getPrunedColList(Operator op) { + return prunedColLists.get(op); + } + + public HashMap, OpParseContext> getOpToParseCtxMap() { + return opToParseCtxMap; + } + + public Map, List> getPrunedColLists() { + return prunedColLists; + } + + public List genColLists(Operator curOp) throws SemanticException { + List colList = new ArrayList(); + if(curOp.getChildOperators() != null) { + for(Operator child: curOp.getChildOperators()) + colList = Utilities.mergeUniqElems(colList, prunedColLists.get(child)); + } + return colList; + } + + public List getColsFromSelectExpr(SelectOperator op) { + List cols = new ArrayList(); + selectDesc conf = op.getConf(); + ArrayList exprList = conf.getColList(); + for (exprNodeDesc expr : exprList) + cols = Utilities.mergeUniqElems(cols, expr.getCols()); + return cols; + } + + public List getSelectColsFromChildren(SelectOperator op, List colList) { + List cols = new ArrayList(); + selectDesc conf = op.getConf(); + ArrayList selectExprs = conf.getColList(); + + for (String col : colList) { + // col is the internal name i.e. position within the expression list + exprNodeDesc expr = selectExprs.get(Integer.parseInt(col)); + cols = Utilities.mergeUniqElems(cols, expr.getCols()); + } + return cols; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (working copy) @@ -23,27 +23,29 @@ import java.io.Serializable; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.parse.OperatorProcessor; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.plan.mapredWork; -import org.apache.hadoop.hive.ql.metadata.*; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.parse.SemanticException; /** * Processor for the rule - table scan followed by reduce sink */ -public class GenMRFileSink1 implements OperatorProcessor { +public class GenMRFileSink1 implements NodeProcessor { public GenMRFileSink1() { } /** * File Sink Operator encountered - * @param op the file sink operator encountered + * @param nd the file sink operator encountered * @param opProcCtx context */ - public void process(FileSinkOperator op, OperatorProcessorContext opProcCtx) throws SemanticException { + public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException { + FileSinkOperator op = (FileSinkOperator)nd; GenMRProcContext ctx = (GenMRProcContext)opProcCtx; boolean ret = false; @@ -81,13 +83,4 @@ } } } - - /** - * @param op the operator encountered - * @param opProcCtx context - */ - public void process(Operator op, OperatorProcessorContext opProcCtx) throws SemanticException { - // should never be called - assert false; - } } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (working copy) @@ -401,4 +401,12 @@ return colLists; } + + /** + * @return the name of the operator + */ + @Override + public String getName() { + return new String("GBY"); + } } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java (working copy) @@ -52,7 +52,7 @@ * The operator name for this operator type. This is used to construct the rule for an operator * @return the operator name **/ - public String getOperatorName() { + public String getName() { return new String("TS"); } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (working copy) @@ -21,7 +21,9 @@ import java.io.Serializable; import java.util.HashMap; import java.util.List; +import java.util.Vector; import java.util.Map; +import org.apache.hadoop.hive.ql.lib.Node; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -36,7 +38,7 @@ /** * Base operator implementation **/ -public abstract class Operator implements Serializable { +public abstract class Operator implements Serializable, Node { // Bean methods @@ -55,6 +57,23 @@ return childOperators; } + /** + * Implements the getChildren function for the Node Interface. + */ + public Vector getChildren() { + + if (getChildOperators() == null) { + return null; + } + + Vector ret_vec = new Vector(); + for(Operator op: getChildOperators()) { + ret_vec.add(op); + } + + return ret_vec; + } + public void setParentOperators(List> parentOperators) { this.parentOperators = parentOperators; } @@ -288,10 +307,10 @@ } /** - * returns the name of the operator - specific subclasses can override the name as and when needed + * Implements the getName function for the Node Interface. * @return the name of the operator */ - public String getOperatorName() { + public String getName() { return new String("OP"); } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java (working copy) @@ -73,4 +73,13 @@ conditionInspectableObject.o.getClass().getName()); } } + + /** + * @return the name of the operator + */ + @Override + public String getName() { + return new String("FIL"); + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (working copy) @@ -200,7 +200,7 @@ /** * @return the name of the operator */ - public String getOperatorName() { + public String getName() { return new String("RS"); } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (working copy) @@ -168,7 +168,7 @@ /** * @return the name of the operator */ - public String getOperatorName() { + public String getName() { return new String("FS"); } } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java (working copy) @@ -87,4 +87,12 @@ } forward(output, outputObjectInspector); } + + /** + * @return the name of the operator + */ + @Override + public String getName() { + return new String("SEL"); + } } Index: ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java (working copy) @@ -20,13 +20,21 @@ package org.apache.hadoop.hive.ql.tools; import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.TreeSet; -import org.antlr.runtime.tree.CommonTree; -import org.apache.hadoop.hive.ql.parse.ASTEvent; -import org.apache.hadoop.hive.ql.parse.ASTEventProcessor; -import org.apache.hadoop.hive.ql.parse.DefaultASTEventDispatcher; -import org.apache.hadoop.hive.ql.parse.DefaultASTProcessor; +import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; +import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; +import org.apache.hadoop.hive.ql.lib.Dispatcher; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.lib.GraphWalker; +import org.apache.hadoop.hive.ql.lib.Rule; +import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.ParseDriver; import org.apache.hadoop.hive.ql.parse.ParseException; @@ -40,7 +48,7 @@ * Later we can expand to add join tables etc. * */ -public class LineageInfo implements ASTEventProcessor { +public class LineageInfo implements NodeProcessor { /** * Stores input tables in sql @@ -66,44 +74,50 @@ return OutputTableList; } - /* (non-Javadoc) - * @see org.apache.hadoop.hive.ql.parse.ASTEventProcessor#process(org.antlr.runtime.tree.CommonTree) + /** + * Implements the process method for the NodeProcessor interface. */ - public void process(CommonTree pt) { + @Override + public void process(Node nd, NodeProcessorCtx procCtx) + throws SemanticException { + ASTNode pt = (ASTNode)nd; + switch (pt.getToken().getType()) { - switch (pt.getToken().getType()) { + case HiveParser.TOK_DESTINATION: { + if (pt.getChild(0).getType() == HiveParser.TOK_TAB) { + OutputTableList.add(pt.getChild(0).getChild(0).getText()) ; + } - case HiveParser.TOK_DESTINATION: { - if (pt.getChild(0).getType() == HiveParser.TOK_TAB) { - OutputTableList.add(pt.getChild(0).getChild(0).getText()) ; - } - - } - break; - case HiveParser.TOK_FROM: { - CommonTree tabRef = (CommonTree) pt.getChild(0); - String table_name = tabRef.getChild(0).getText(); - inputTableList.add(table_name); - } - break; - } - } + } + break; + case HiveParser.TOK_FROM: { + if (((ASTNode)pt.getChild(0)).getToken().getType() == HiveParser.TOK_TABREF) { + ASTNode tabRef = (ASTNode) pt.getChild(0); + String table_name = tabRef.getChild(0).getText(); + inputTableList.add(table_name); + } + } + break; + } + + } + /** * parses given query and gets the lineage info. * @param query * @throws ParseException */ - public void getLineageInfo(String query) throws ParseException + public void getLineageInfo(String query) throws ParseException, SemanticException { /* * Get the AST tree */ ParseDriver pd = new ParseDriver(); - CommonTree tree = pd.parse(query); + ASTNode tree = pd.parse(query); while ((tree.getToken() == null) && (tree.getChildCount() > 0)) { - tree = (CommonTree) tree.getChild(0); + tree = (ASTNode) tree.getChild(0); } /* @@ -111,14 +125,19 @@ */ inputTableList.clear(); OutputTableList.clear(); - DefaultASTEventDispatcher dispatcher = new DefaultASTEventDispatcher(); - dispatcher.register(ASTEvent.SRC_TABLE, this); - dispatcher.register(ASTEvent.DESTINATION, this); + + // create a walker which walks the tree in a DFS manner while maintaining the operator stack. The dispatcher + // generates the plan from the operator tree + Map rules = new LinkedHashMap(); - DefaultASTProcessor eventProcessor = new DefaultASTProcessor(); - - eventProcessor.setDispatcher(dispatcher); - eventProcessor.process(tree); + // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along + Dispatcher disp = new DefaultRuleDispatcher(this, rules, null); + GraphWalker ogw = new DefaultGraphWalker(disp); + + // Create a list of topop nodes + ArrayList topNodes = new ArrayList(); + topNodes.add(tree); + ogw.startWalking(topNodes); } public static void main(String[] args) throws IOException, ParseException, Index: ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java (working copy) @@ -16,15 +16,13 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.parse; +package org.apache.hadoop.hive.ql.lib; import java.util.Stack; -import java.util.Iterator; -import java.util.regex.Pattern; import java.util.regex.Matcher; -import java.io.Serializable; +import java.util.regex.Pattern; -import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.SemanticException; /** * Rule interface for Operators @@ -33,7 +31,6 @@ public class RuleRegExp implements Rule { private String ruleName; - private String regExp; private Pattern pattern; /** @@ -44,7 +41,6 @@ **/ public RuleRegExp(String ruleName, String regExp) { this.ruleName = ruleName; - this.regExp = regExp; pattern = Pattern.compile(regExp); } @@ -54,11 +50,11 @@ * @return cost of the function * @throws SemanticException */ - public int cost(Stack> stack) throws SemanticException { - int numElems = stack.size(); + public int cost(Stack stack) throws SemanticException { + int numElems = (stack != null ? stack.size() : 0); String name = new String(); for (int pos = numElems - 1; pos >= 0; pos--) { - name = stack.get(pos).getOperatorName().concat(name); + name = stack.get(pos).getName().concat(name); Matcher m = pattern.matcher(name); if (m.matches()) return m.group().length(); Property changes on: ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java ___________________________________________________________________ Added: svn:mergeinfo Index: ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java (working copy) @@ -16,21 +16,12 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.parse; +package org.apache.hadoop.hive.ql.lib; -import java.io.Serializable; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.ClassNotFoundException; -import java.util.List; import java.util.Map; import java.util.Stack; -import java.lang.ClassNotFoundException; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext; -import org.apache.hadoop.hive.ql.exec.OperatorFactory; -import org.apache.hadoop.hive.ql.exec.OperatorFactory.opTuple; +import org.apache.hadoop.hive.ql.parse.SemanticException; /** * Dispatches calls to relevant method in processor. The user registers various rules with the dispatcher, and @@ -38,95 +29,52 @@ */ public class DefaultRuleDispatcher implements Dispatcher { - private Map opProcRules; - private OperatorProcessorContext opProcCtx; - private OperatorProcessor defaultProc; + private Map procRules; + private NodeProcessorCtx procCtx; + private NodeProcessor defaultProc; /** * constructor * @param defaultProc defualt processor to be fired if no rule matches - * @param opp operator processor that handles actual processing of the node - * @param opProcCtx operator processor context, which is opaque to the dispatcher + * @param rules operator processor that handles actual processing of the node + * @param procCtx operator processor context, which is opaque to the dispatcher */ - public DefaultRuleDispatcher(OperatorProcessor defaultProc, - Map opp, OperatorProcessorContext opProcCtx) { + public DefaultRuleDispatcher(NodeProcessor defaultProc, + Map rules, NodeProcessorCtx procCtx) { this.defaultProc = defaultProc; - this.opProcRules = opp; - this.opProcCtx = opProcCtx; + this.procRules = rules; + this.procCtx = procCtx; } /** * dispatcher function - * @param op operator to process - * @param opStack the operators encountered so far + * @param nd operator to process + * @param ndStack the operators encountered so far * @throws SemanticException */ - public void dispatch(Operator op, Stack> opStack) + public void dispatch(Node nd, Stack ndStack) throws SemanticException { // find the firing rule // find the rule from the stack specified Rule rule = null; int minCost = Integer.MAX_VALUE; - for (Rule r : opProcRules.keySet()) { - int cost = r.cost(opStack); + for (Rule r : procRules.keySet()) { + int cost = r.cost(ndStack); if ((cost >= 0) && (cost <= minCost)) { minCost = cost; rule = r; } } - OperatorProcessor proc; + NodeProcessor proc; if (rule == null) proc = defaultProc; else - proc = opProcRules.get(rule); + proc = procRules.get(rule); - // If the processor has registered a process method for the particular operator, invoke it. - // Otherwise implement the generic function, which would definitely be implemented - for(opTuple opt : OperatorFactory.opvec) { - if(opt.opClass.isInstance(op)) { - Method pcall; - try { - pcall = proc.getClass().getMethod("process", opt.opClass, - Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext")); - pcall.invoke(proc, op, opProcCtx); - return; - } catch (SecurityException e) { - assert false; - } catch (NoSuchMethodException e) { - assert false; - } catch (IllegalArgumentException e) { - assert false; - } catch (IllegalAccessException e) { - assert false; - } catch (InvocationTargetException e) { - throw new SemanticException(e.getTargetException()); - } catch (ClassNotFoundException e) { - assert false; - } - } - } - - try { - // no method found - invoke the generic function - Method pcall = proc.getClass().getMethod("process", Class.forName("org.apache.hadoop.hive.ql.exec.Operator"), - Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext")); - pcall.invoke(proc, ((Operator)op), opProcCtx); - - } catch (SecurityException e) { - assert false; - } catch (NoSuchMethodException e) { - assert false; - } catch (IllegalArgumentException e) { - assert false; - } catch (IllegalAccessException e) { - assert false; - } catch (InvocationTargetException e) { - throw new SemanticException(e.getTargetException()); - } catch (ClassNotFoundException e) { - assert false; - } + // Call the process function + proc.process(nd, procCtx); } } Property changes on: ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java ___________________________________________________________________ Added: svn:mergeinfo Index: ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java (working copy) @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.optimizer; +package org.apache.hadoop.hive.ql.lib; /** * Operator Processor Context */ -public abstract class OperatorProcessorContext { +public abstract class NodeProcessorCtx { } Property changes on: ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java ___________________________________________________________________ Added: svn:mergeinfo Index: ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java (working copy) @@ -15,25 +15,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hive.ql.parse; +package org.apache.hadoop.hive.ql.lib; -import java.io.Serializable; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext; +import org.apache.hadoop.hive.ql.parse.SemanticException; /** * Base class for processing operators which is no-op. The specific processors can register their own context with * the dispatcher. */ -public interface OperatorProcessor { +public interface NodeProcessor { /** * generic process for all ops that don't have specific implementations - * @param op operator to process - * @param opProcCtx operator processor context + * @param nd operator to process + * @param procCtx operator processor context * @throws SemanticException */ - public void process(Operator op, OperatorProcessorContext opProcCtx) + public void process(Node nd, NodeProcessorCtx procCtx) throws SemanticException; } Property changes on: ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java ___________________________________________________________________ Added: svn:mergeinfo Index: ql/src/java/org/apache/hadoop/hive/ql/lib/Node.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lib/Node.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/lib/Node.java (revision 0) @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.lib; + +import java.util.Vector; + +/** + * This interface defines the functions needed by the walkers and dispatchers. + * These are implemented by the node of the graph that needs to be walked. + */ +public interface Node { + + /** + * Gets the vector of children nodes. This is used in the graph walker algorithms. + * + * @return Vector + */ + public Vector getChildren(); + + /** + * Gets the name of the node. This is used in the rule dispatchers. + * + * @return String + */ + public String getName(); +} Index: ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java (working copy) @@ -16,12 +16,11 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.parse; +package org.apache.hadoop.hive.ql.lib; -import java.io.Serializable; import java.util.Stack; -import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.SemanticException; /** * Dispatcher interface for Operators @@ -31,10 +30,10 @@ /** * dispatcher function - * @param op operator to process + * @param nd operator to process * @param Stack operator stack to process * @throws SemanticException */ - public abstract void dispatch(Operator op, Stack> stack) + public abstract void dispatch(Node nd, Stack stack) throws SemanticException; } Property changes on: ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java ___________________________________________________________________ Added: svn:mergeinfo Index: ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java (working copy) @@ -16,11 +16,10 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.parse; +package org.apache.hadoop.hive.ql.lib; import java.util.Stack; -import java.io.Serializable; -import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.SemanticException; /** * Rule interface for Operators @@ -32,7 +31,7 @@ * @return the cost of the rule - the lower the cost, the better the rule matches * @throws SemanticException */ - public int cost(Stack> stack) throws SemanticException; + public int cost(Stack stack) throws SemanticException; /** * @return the name of the rule - may be useful for debugging Property changes on: ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java ___________________________________________________________________ Added: svn:mergeinfo Index: ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java (working copy) @@ -16,25 +16,24 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.parse; +package org.apache.hadoop.hive.ql.lib; -import java.io.Serializable; import java.util.Collection; -import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.SemanticException; /** * Interface for operator graph walker. */ -public interface OpGraphWalker { +public interface GraphWalker { /** * starting point for walking. * - * @param startOps list of starting operators + * @param startNodes list of starting operators * @throws SemanticException */ - public abstract void startWalking(Collection> startOps) + public abstract void startWalking(Collection startNodes) throws SemanticException; } \ No newline at end of file Property changes on: ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java ___________________________________________________________________ Added: svn:mergeinfo Index: ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java (working copy) @@ -16,9 +16,8 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.parse; +package org.apache.hadoop.hive.ql.lib; -import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -26,39 +25,44 @@ import java.util.Set; import java.util.Stack; -import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; +import org.apache.hadoop.hive.ql.exec.ScriptOperator; +import org.apache.hadoop.hive.ql.exec.SelectOperator; +import org.apache.hadoop.hive.ql.parse.SemanticException; /** * base class for operator graph walker * this class takes list of starting ops and walks them one by one. it maintains list of walked * operators (dispatchedList) and a list of operators that are discovered but not yet dispatched */ -public abstract class DefaultOpGraphWalker implements OpGraphWalker { +public class DefaultGraphWalker implements GraphWalker { - List> toWalk = new ArrayList>(); - Set> dispatchedList = new HashSet>(); - Dispatcher dispatcher; + protected Stack opStack; + private List toWalk = new ArrayList(); + private Set dispatchedList = new HashSet(); + private Dispatcher dispatcher; /** * Constructor * @param ctx graph of operators to walk * @param disp dispatcher to call for each op encountered */ - public DefaultOpGraphWalker(Dispatcher disp) { + public DefaultGraphWalker(Dispatcher disp) { this.dispatcher = disp; - } + opStack = new Stack(); + } /** * @return the toWalk */ - public List> getToWalk() { + public List getToWalk() { return toWalk; } /** * @return the doneList */ - public Set> getDispatchedList() { + public Set getDispatchedList() { return dispatchedList; } @@ -68,25 +72,41 @@ * @param opStack stack of operators encountered * @throws SemanticException */ - public void dispatch(Operator op, Stack opStack) throws SemanticException { - this.dispatcher.dispatch(op, opStack); - this.dispatchedList.add(op); + public void dispatch(Node nd, Stack ndStack) throws SemanticException { + this.dispatcher.dispatch(nd, ndStack); + this.dispatchedList.add(nd); } /** * starting point for walking * @throws SemanticException */ - public void startWalking(Collection> startOps) throws SemanticException { - toWalk.addAll(startOps); + public void startWalking(Collection startNodes) throws SemanticException { + toWalk.addAll(startNodes); while(toWalk.size() > 0) walk(toWalk.remove(0)); } /** * walk the current operator and its descendants - * @param op current operator in the graph + * @param nd current operator in the graph * @throws SemanticException */ - public abstract void walk(Operator op) throws SemanticException; + public void walk(Node nd) throws SemanticException { + opStack.push(nd); + + if((nd.getChildren() == null) + || getDispatchedList().containsAll(nd.getChildren())) { + // all children are done or no need to walk the children + dispatch(nd, opStack); + opStack.pop(); + return; + } + // move all the children to the front of queue + getToWalk().removeAll(nd.getChildren()); + getToWalk().addAll(0, nd.getChildren()); + // add self to the end of the queue + getToWalk().add(nd); + opStack.pop(); + } } Property changes on: ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java ___________________________________________________________________ Added: svn:mergeinfo Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java (working copy) @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import org.antlr.runtime.tree.CommonTree; - -/** - * Dispatches ParseTreeEvent to the appropriate ParseTreeEventProcessor - */ -public interface ASTEventDispatcher { - - /** - * Registers the event processor with the event - * - * @param evt The parse tree event - * @param evt_p The associated parse tree event processor - */ - void register(ASTEvent evt, ASTEventProcessor evt_p); - - /** - * Dispatches the parse tree event to a registered event processor - * - * @param evt The parse tree event to dispatch - * @param pt The parse subtree to dispatch to the event processor - */ - void dispatch(ASTEvent evt, CommonTree pt); -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java (working copy) @@ -20,8 +20,6 @@ import java.util.Vector; -import org.antlr.runtime.tree.CommonTree; - /** * Internal representation of the join tree * @@ -38,10 +36,10 @@ private boolean noOuterJoin; // join conditions - private Vector> expressions; + private Vector> expressions; // filters - private Vector> filters; + private Vector> filters; /** * constructor @@ -80,11 +78,11 @@ this.leftAliases = leftAliases; } - public Vector> getExpressions() { + public Vector> getExpressions() { return expressions; } - public void setExpressions(Vector> expressions) { + public void setExpressions(Vector> expressions) { this.expressions = expressions; } @@ -131,14 +129,14 @@ /** * @return the filters */ - public Vector> getFilters() { + public Vector> getFilters() { return filters; } /** * @param filters the filters to set */ - public void setFilters(Vector> filters) { + public void setFilters(Vector> filters) { this.filters = filters; } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java (working copy) @@ -20,8 +20,6 @@ import java.util.*; -import org.antlr.runtime.tree.*; - import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; import org.apache.hadoop.hive.ql.metadata.*; @@ -89,7 +87,7 @@ * @throws SemanticException */ @SuppressWarnings("nls") - private exprNodeDesc genExprNodeDesc(CommonTree expr) + private exprNodeDesc genExprNodeDesc(ASTNode expr) throws SemanticException { // We recursively create the exprNodeDesc. Base cases: when we encounter // a column ref, we convert that into an exprNodeColumnDesc; when we encounter @@ -117,7 +115,7 @@ } else { colName = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText()); - tabAlias = SemanticAnalyzer.getTabAliasForCol(this.metaData, colName, (CommonTree)expr.getChild(0)); + tabAlias = SemanticAnalyzer.getTabAliasForCol(this.metaData, colName, (ASTNode)expr.getChild(0)); } // Set value to null if it's not partition column @@ -150,7 +148,7 @@ int childrenBegin = (isFunction ? 1 : 0); ArrayList children = new ArrayList(expr.getChildCount() - childrenBegin); for (int ci=childrenBegin; ci> startOps) - throws SemanticException; - -} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java (working copy) @@ -18,21 +18,17 @@ package org.apache.hadoop.hive.ql.parse; -import java.io.Serializable; -import java.util.List; import java.util.Stack; -import java.util.regex.Pattern; -import java.util.regex.Matcher; - -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import java.util.Vector; import org.apache.hadoop.hive.ql.exec.*; +import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; +import org.apache.hadoop.hive.ql.lib.Dispatcher; +import org.apache.hadoop.hive.ql.lib.Node; /** * Walks the operator tree in pre order fashion */ -public class GenMapRedWalker extends DefaultOpGraphWalker { - private Stack> opStack; +public class GenMapRedWalker extends DefaultGraphWalker { /** * constructor of the walker - the dispatcher is passed @@ -40,30 +36,29 @@ */ public GenMapRedWalker(Dispatcher disp) { super(disp); - opStack = new Stack>(); } /** * Walk the given operator - * @param op operator being walked + * @param nd operator being walked */ @Override - public void walk(Operator op) throws SemanticException { - List> children = op.getChildOperators(); + public void walk(Node nd) throws SemanticException { + Vector children = nd.getChildren(); // maintain the stack of operators encountered - opStack.push(op); - dispatch(op, opStack); + opStack.push(nd); + dispatch(nd, opStack); // kids of reduce sink operator need not be traversed again if ((children == null) || - ((op instanceof ReduceSinkOperator) && (getDispatchedList().containsAll(children)))) { + ((nd instanceof ReduceSinkOperator) && (getDispatchedList().containsAll(children)))) { opStack.pop(); return; } // move all the children to the front of queue - for (Operator ch : children) + for (Node ch : children) walk(ch); // done with this operator Index: ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (working copy) @@ -26,7 +26,6 @@ import java.util.HashMap; import java.util.List; -import org.antlr.runtime.tree.CommonTree; import org.antlr.runtime.tree.Tree; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FileStatus; @@ -156,7 +155,7 @@ } @Override - public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException { + public void analyzeInternal(ASTNode ast, Context ctx) throws SemanticException { isLocal = isOverWrite = false; Tree from_t = ast.getChild(0); Tree table_t = ast.getChild(1); @@ -185,7 +184,7 @@ } // initialize destination table/partition - tableSpec ts = new tableSpec(db, (CommonTree) table_t, true); + tableSpec ts = new tableSpec(db, (ASTNode) table_t, true); URI toURI = (ts.partHandle != null) ? ts.partHandle.getDataLocation() : ts.tableHandle.getDataLocation(); // make sure the arguments make sense Index: ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (working copy) @@ -33,86 +33,86 @@ private boolean isSubQ; private String alias; - private CommonTree joinExpr; - private HashMap aliasToSrc; - private HashMap nameToDest; + private ASTNode joinExpr; + private HashMap aliasToSrc; + private HashMap nameToDest; private HashMap nameToSample; - private HashMap destToSelExpr; - private HashMap destToWhereExpr; - private HashMap destToGroupby; + private HashMap destToSelExpr; + private HashMap destToWhereExpr; + private HashMap destToGroupby; /** * ClusterBy is a short name for both DistributeBy and SortBy. */ - private HashMap destToClusterby; + private HashMap destToClusterby; /** * DistributeBy controls the hashcode of the row, which determines which reducer * the rows will go to. */ - private HashMap destToDistributeby; + private HashMap destToDistributeby; /** * SortBy controls the reduce keys, which affects the order of rows * that the reducer receives. */ - private HashMap destToSortby; + private HashMap destToSortby; private HashMap destToLimit; private int outerQueryLimit; // used by GroupBy - private HashMap > destToAggregationExprs; - private HashMap destToDistinctFuncExpr; + private HashMap > destToAggregationExprs; + private HashMap destToDistinctFuncExpr; @SuppressWarnings("unused") private static final Log LOG = LogFactory.getLog(QBParseInfo.class.getName()); public QBParseInfo(String alias, boolean isSubQ) { - this.aliasToSrc = new HashMap(); - this.nameToDest = new HashMap(); + this.aliasToSrc = new HashMap(); + this.nameToDest = new HashMap(); this.nameToSample = new HashMap(); - this.destToSelExpr = new HashMap(); - this.destToWhereExpr = new HashMap(); - this.destToGroupby = new HashMap(); - this.destToClusterby = new HashMap(); - this.destToDistributeby = new HashMap(); - this.destToSortby = new HashMap(); + this.destToSelExpr = new HashMap(); + this.destToWhereExpr = new HashMap(); + this.destToGroupby = new HashMap(); + this.destToClusterby = new HashMap(); + this.destToDistributeby = new HashMap(); + this.destToSortby = new HashMap(); this.destToLimit = new HashMap(); - this.destToAggregationExprs = new HashMap >(); - this.destToDistinctFuncExpr = new HashMap(); + this.destToAggregationExprs = new HashMap >(); + this.destToDistinctFuncExpr = new HashMap(); this.alias = alias; this.isSubQ = isSubQ; this.outerQueryLimit = -1; } - public void setAggregationExprsForClause(String clause, HashMap aggregationTrees) { + public void setAggregationExprsForClause(String clause, HashMap aggregationTrees) { this.destToAggregationExprs.put(clause, aggregationTrees); } - public HashMap getAggregationExprsForClause(String clause) { + public HashMap getAggregationExprsForClause(String clause) { return this.destToAggregationExprs.get(clause); } - public void setDistinctFuncExprForClause(String clause, CommonTree ast) { + public void setDistinctFuncExprForClause(String clause, ASTNode ast) { this.destToDistinctFuncExpr.put(clause, ast); } - public CommonTree getDistinctFuncExprForClause(String clause) { + public ASTNode getDistinctFuncExprForClause(String clause) { return this.destToDistinctFuncExpr.get(clause); } - public void setSelExprForClause(String clause, CommonTree ast) { + public void setSelExprForClause(String clause, ASTNode ast) { this.destToSelExpr.put(clause, ast); } - public void setWhrExprForClause(String clause, CommonTree ast) { + public void setWhrExprForClause(String clause, ASTNode ast) { this.destToWhereExpr.put(clause, ast); } - public void setGroupByExprForClause(String clause, CommonTree ast) { + public void setGroupByExprForClause(String clause, ASTNode ast) { this.destToGroupby.put(clause, ast); } - public void setDestForClause(String clause, CommonTree ast) { + public void setDestForClause(String clause, ASTNode ast) { this.nameToDest.put(clause, ast); } @@ -121,7 +121,7 @@ * @param clause the name of the clause * @param ast the abstract syntax tree */ - public void setClusterByExprForClause(String clause, CommonTree ast) { + public void setClusterByExprForClause(String clause, ASTNode ast) { this.destToClusterby.put(clause, ast); } @@ -130,7 +130,7 @@ * @param clause the name of the clause * @param ast the abstract syntax tree */ - public void setDistributeByExprForClause(String clause, CommonTree ast) { + public void setDistributeByExprForClause(String clause, ASTNode ast) { this.destToDistributeby.put(clause, ast); } @@ -139,11 +139,11 @@ * @param clause the name of the clause * @param ast the abstract syntax tree */ - public void setSortByExprForClause(String clause, CommonTree ast) { + public void setSortByExprForClause(String clause, ASTNode ast) { this.destToSortby.put(clause, ast); } - public void setSrcForAlias(String alias, CommonTree ast) { + public void setSrcForAlias(String alias, ASTNode ast) { this.aliasToSrc.put(alias.toLowerCase(), ast); } @@ -155,23 +155,23 @@ return this.nameToDest.keySet(); } - public CommonTree getDestForClause(String clause) { + public ASTNode getDestForClause(String clause) { return this.nameToDest.get(clause); } - public CommonTree getWhrForClause(String clause) { + public ASTNode getWhrForClause(String clause) { return this.destToWhereExpr.get(clause); } - public HashMap getDestToWhereExpr() { + public HashMap getDestToWhereExpr() { return destToWhereExpr; } - public CommonTree getGroupByForClause(String clause) { + public ASTNode getGroupByForClause(String clause) { return this.destToGroupby.get(clause); } - public CommonTree getSelForClause(String clause) { + public ASTNode getSelForClause(String clause) { return this.destToSelExpr.get(clause); } @@ -180,7 +180,7 @@ * @param clause the name of the clause * @return the abstract syntax tree */ - public CommonTree getClusterByForClause(String clause) { + public ASTNode getClusterByForClause(String clause) { return this.destToClusterby.get(clause); } @@ -189,7 +189,7 @@ * @param clause the name of the clause * @return the abstract syntax tree */ - public CommonTree getDistributeByForClause(String clause) { + public ASTNode getDistributeByForClause(String clause) { return this.destToDistributeby.get(clause); } @@ -198,11 +198,11 @@ * @param clause the name of the clause * @return the abstract syntax tree */ - public CommonTree getSortByForClause(String clause) { + public ASTNode getSortByForClause(String clause) { return this.destToSortby.get(clause); } - public CommonTree getSrcForAlias(String alias) { + public ASTNode getSrcForAlias(String alias) { return this.aliasToSrc.get(alias.toLowerCase()); } @@ -214,11 +214,11 @@ return this.isSubQ; } - public CommonTree getJoinExpr() { + public ASTNode getJoinExpr() { return this.joinExpr; } - public void setJoinExpr(CommonTree joinExpr) { + public void setJoinExpr(ASTNode joinExpr) { this.joinExpr = joinExpr; } @@ -260,40 +260,40 @@ (!destToClusterby.isEmpty())) return false; - Iterator>> aggrIter = destToAggregationExprs.entrySet().iterator(); + Iterator>> aggrIter = destToAggregationExprs.entrySet().iterator(); while (aggrIter.hasNext()) { - HashMap h = aggrIter.next().getValue(); + HashMap h = aggrIter.next().getValue(); if ((h != null) && (!h.isEmpty())) return false; } if (!destToDistinctFuncExpr.isEmpty()) { - Iterator> distn = destToDistinctFuncExpr.entrySet().iterator(); + Iterator> distn = destToDistinctFuncExpr.entrySet().iterator(); while (distn.hasNext()) { - CommonTree ct = distn.next().getValue(); + ASTNode ct = distn.next().getValue(); if (ct != null) return false; } } - Iterator> iter = nameToDest.entrySet().iterator(); + Iterator> iter = nameToDest.entrySet().iterator(); while (iter.hasNext()) { - Map.Entry entry = iter.next(); - CommonTree v = entry.getValue(); - if (!(((CommonTree)v.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) + Map.Entry entry = iter.next(); + ASTNode v = entry.getValue(); + if (!(((ASTNode)v.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) return false; } iter = destToSelExpr.entrySet().iterator(); while (iter.hasNext()) { - Map.Entry entry = iter.next(); - CommonTree selExprList = entry.getValue(); + Map.Entry entry = iter.next(); + ASTNode selExprList = entry.getValue(); // Iterate over the selects for (int i = 0; i < selExprList.getChildCount(); ++i) { // list of the columns - CommonTree selExpr = (CommonTree) selExprList.getChild(i); - CommonTree sel = (CommonTree)selExpr.getChild(0); + ASTNode selExpr = (ASTNode) selExprList.getChild(i); + ASTNode sel = (ASTNode)selExpr.getChild(0); if (sel.getToken().getType() != HiveParser.TOK_ALLCOLREF) return false; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java (working copy) @@ -21,12 +21,12 @@ import java.io.PrintStream; import java.io.Serializable; import java.util.HashMap; -import java.util.Stack; - import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext; +import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -public class PrintOpTreeProcessor implements OperatorProcessor { +public class PrintOpTreeProcessor implements NodeProcessor { private PrintStream out; private HashMap, Integer> opMap = new HashMap, Integer>(); @@ -70,7 +70,8 @@ return ret.toString(); } - public void process(Operator op, OperatorProcessorContext ctx) throws SemanticException { + public void process(Node nd, NodeProcessorCtx ctx) throws SemanticException { + Operator op = (Operator)nd; if (opMap.get(op) == null) { opMap.put(op, curNum++); } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultOpGraphWalker.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultOpGraphWalker.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultOpGraphWalker.java (working copy) @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.Stack; - -import org.apache.hadoop.hive.ql.exec.Operator; - -/** - * base class for operator graph walker - * this class takes list of starting ops and walks them one by one. it maintains list of walked - * operators (dispatchedList) and a list of operators that are discovered but not yet dispatched - */ -public abstract class DefaultOpGraphWalker implements OpGraphWalker { - - List> toWalk = new ArrayList>(); - Set> dispatchedList = new HashSet>(); - Dispatcher dispatcher; - - /** - * Constructor - * @param ctx graph of operators to walk - * @param disp dispatcher to call for each op encountered - */ - public DefaultOpGraphWalker(Dispatcher disp) { - this.dispatcher = disp; - } - - /** - * @return the toWalk - */ - public List> getToWalk() { - return toWalk; - } - - /** - * @return the doneList - */ - public Set> getDispatchedList() { - return dispatchedList; - } - - /** - * Dispatch the current operator - * @param op operator being walked - * @param opStack stack of operators encountered - * @throws SemanticException - */ - public void dispatch(Operator op, Stack opStack) throws SemanticException { - this.dispatcher.dispatch(op, opStack); - this.dispatchedList.add(op); - } - - /** - * starting point for walking - * @throws SemanticException - */ - public void startWalking(Collection> startOps) throws SemanticException { - toWalk.addAll(startOps); - while(toWalk.size() > 0) - walk(toWalk.remove(0)); - } - - /** - * walk the current operator and its descendants - * @param op current operator in the graph - * @throws SemanticException - */ - public abstract void walk(Operator op) throws SemanticException; -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java (working copy) @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.ql.parse; -import org.antlr.runtime.tree.CommonTree; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; @@ -36,7 +34,7 @@ super(conf); } - public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException { + public void analyzeInternal(ASTNode ast, Context ctx) throws SemanticException { String functionName = ast.getChild(0).getText(); String className = unescapeSQLString(ast.getChild(1).getText()); createFunctionDesc desc = new createFunctionDesc(functionName, className); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java (working copy) @@ -245,12 +245,24 @@ } - public CommonTree parse(String command) throws ParseException { + /** + * Tree adaptor for making antlr return ASTNodes instead of CommonTree nodes + * so that the graph walking algorithms and the rules framework defined in + * ql.lib can be used with the AST Nodes. + */ + static final TreeAdaptor adaptor = new CommonTreeAdaptor() { + public Object create(Token payload) { + return new ASTNode(payload); + } + }; + + public ASTNode parse(String command) throws ParseException { LOG.info("Parsing command: " + command); HiveLexerX lexer = new HiveLexerX(new ANTLRNoCaseStringStream(command)); TokenStream tokens = new TokenRewriteStream(lexer); HiveParserX parser = new HiveParserX(tokens); + parser.setTreeAdaptor(adaptor); HiveParser.statement_return r = null; try { r = parser.statement(); @@ -268,7 +280,7 @@ throw new ParseException(parser.getErrors()); } - return (CommonTree)r.getTree(); + return (ASTNode)r.getTree(); } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/TopoWalker.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/TopoWalker.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TopoWalker.java (working copy) @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import java.io.Serializable; - -import org.apache.hadoop.hive.ql.exec.Operator; - -/** - * walks the tree in toplogically sorted order - */ -public class TopoWalker extends DefaultOpGraphWalker { - - public TopoWalker(Dispatcher disp) { - super(disp); - } - - @Override - public void walk(Operator op) throws SemanticException { - // the dispatcher does not care about the stack - so dont maintin it - dispatch(op, null); - if(op.getChildOperators() != null) { - for(Operator child : op.getChildOperators()) { - if(getDispatchedList().containsAll(child.getParentOperators())) { - walk(child); - } - } - } - } - -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java (working copy) @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -/** - * Enumeration that encapsulates the various event types that are seen - * while processing the parse tree (in an implementation of the ParseTreeProcessor). - * These event types are used to register the different event processors with - * the parse tree processor. - * - */ -public enum ASTEvent { - - /** - * Query event - */ - QUERY("QUERY"), - - /** - * Union - */ - UNION("UNION"), - - /** - * Source Table (table in the from clause) - */ - SRC_TABLE("SRC_TABLE"), - - /** - * Any type of Destination (this fires for hdfs directory, local directory and table) - */ - DESTINATION("DESTINATION"), - - /** - * Select clause - */ - SELECT_CLAUSE("SELECT_CLAUSE"), - - /** - * Join clause - */ - JOIN_CLAUSE("JOIN_CLAUSE"), - - /** - * Where clause - */ - WHERE_CLAUSE("WHERE_CLAUSE"), - - /** - * CLusterby clause - */ - CLUSTERBY_CLAUSE("CLUSTERBY_CLAUSE"), - - /** - * Group by clause - */ - GROUPBY_CLAUSE("GROUPBY_CLAUSE"), - - /** - * Limit clause - */ - LIMIT_CLAUSE("LIMIT_CLAUSE"), - - /** - * Subquery - */ - SUBQUERY("SUBQUERY"); - - /** - * The name of the event (string representation of the event) - */ - private final String name; - - /** - * Constructs the event - * - * @param name The name(String representation of the event) - */ - ASTEvent(String name) { - this.name = name; - } - - /** - * String representation of the event - * - * @return String - */ - @Override - public String toString() { - return name; - } -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultDispatcher.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultDispatcher.java (working copy) @@ -1,103 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import java.io.Serializable; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.List; -import java.util.Stack; -import java.lang.ClassNotFoundException; - -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.OperatorFactory; -import org.apache.hadoop.hive.ql.exec.OperatorFactory.opTuple; - -/** - * Dispatches calls to relevant method in processor - */ -public class DefaultDispatcher implements Dispatcher { - - private OperatorProcessor opProcessor; - - /** - * constructor - * @param opp operator processor that handles actual processing of the node - */ - public DefaultDispatcher(OperatorProcessor opp) { - this.opProcessor = opp; - } - - /** - * dispatcher function - * @param op operator to process - * @param opStack the operators encountered so far - * @throws SemanticException - */ - public void dispatch(Operator op, Stack> opStack) - throws SemanticException { - - // If the processor has registered a process method for the particular operator, invoke it. - // Otherwise implement the generic function, which would definitely be implemented - for(opTuple opt : OperatorFactory.opvec) { - if(opt.opClass.isInstance(op)) { - Method pcall; - try { - pcall = opProcessor.getClass().getMethod("process", opt.opClass, - Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext")); - pcall.invoke(opProcessor, op, null); - return; - } catch (SecurityException e) { - assert false; - } catch (NoSuchMethodException e) { - assert false; - } catch (IllegalArgumentException e) { - assert false; - } catch (IllegalAccessException e) { - assert false; - } catch (InvocationTargetException e) { - throw new SemanticException(e.getTargetException()); - } catch (ClassNotFoundException e) { - assert false; - } - } - } - - try { - // no method found - invoke the generic function - Method pcall = opProcessor.getClass().getMethod("process", Class.forName("org.apache.hadoop.hive.ql.exec.Operator"), - Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext")); - - pcall.invoke(opProcessor, ((Operator)op), null); - return; - } catch (SecurityException e) { - assert false; - } catch (NoSuchMethodException e) { - assert false; - } catch (IllegalArgumentException e) { - assert false; - } catch (IllegalAccessException e) { - assert false; - } catch (InvocationTargetException e) { - throw new SemanticException(e.getTargetException()); - } catch (ClassNotFoundException e) { - assert false; - } - } -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTProcessor.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTProcessor.java (working copy) @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import org.antlr.runtime.tree.CommonTree; - -/** - * Implementation of a parse tree processor. This processor does a depth first walk - * of the parse tree and calls the associated event processors - */ -public class DefaultASTProcessor implements ASTProcessor { - - /** - * The dispatcher used to dispatch ParseTreeEvents to the ParseTreeEventProcessors - */ - private ASTEventDispatcher dispatcher; - - /** - * Processes the parse tree - * - * @see org.apache.hadoop.hive.ql.parse.ASTProcessor#process(org.antlr.runtime.tree.CommonTree) - */ - @Override - public void process(CommonTree ast) { - - // Base case - if (ast.getToken() == null) { - return; - } - - switch (ast.getToken().getType()) { - case HiveParser.TOK_SELECTDI: - case HiveParser.TOK_SELECT: - dispatcher.dispatch(ASTEvent.SELECT_CLAUSE, ast); - break; - - case HiveParser.TOK_WHERE: - dispatcher.dispatch(ASTEvent.WHERE_CLAUSE, ast); - break; - - case HiveParser.TOK_DESTINATION: - dispatcher.dispatch(ASTEvent.DESTINATION, ast); - break; - - case HiveParser.TOK_FROM: - - // Check if this is a subquery - CommonTree frm = (CommonTree) ast.getChild(0); - if (frm.getToken().getType() == HiveParser.TOK_TABREF) { - dispatcher.dispatch(ASTEvent.SRC_TABLE, ast); - } else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) { - dispatcher.dispatch(ASTEvent.SUBQUERY, ast); - } else if (ParseUtils.isJoinToken(frm)) { - dispatcher.dispatch(ASTEvent.JOIN_CLAUSE, ast); - } - break; - - case HiveParser.TOK_CLUSTERBY: - dispatcher.dispatch(ASTEvent.CLUSTERBY_CLAUSE, ast); - break; - - case HiveParser.TOK_GROUPBY: - dispatcher.dispatch(ASTEvent.GROUPBY_CLAUSE, ast); - break; - - case HiveParser.TOK_LIMIT: - dispatcher.dispatch(ASTEvent.LIMIT_CLAUSE, ast); - break; - default: - break; - } - - // Iterate over the rest of the children - int child_count = ast.getChildCount(); - for (int child_pos = 0; child_pos < child_count; ++child_pos) { - // Recurse - process((CommonTree) ast.getChild(child_pos)); - } - } - - /** - * Sets the dispatcher for the parse tree processor - * - * @see org.apache.hadoop.hive.ql.parse.ASTProcessor#register(org.apache.hadoop.hive.ql.parse.ASTEvent, org.apache.hadoop.hive.ql.parse.ParseTreeEventProcessor) - */ - @Override - public void setDispatcher(ASTEventDispatcher dispatcher) { - - this.dispatcher = dispatcher; - } - -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java (revision 0) @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.util.Vector; +import org.antlr.runtime.tree.CommonTree; +import org.antlr.runtime.Token; +import org.apache.hadoop.hive.ql.lib.Node; + +/** + * @author athusoo + * + */ +public class ASTNode extends CommonTree implements Node { + + /** + * Constructor + * @param t Token for the CommonTree Node + */ + public ASTNode(Token t) { + super(t); + } + + /* (non-Javadoc) + * @see org.apache.hadoop.hive.ql.lib.Node#getChildren() + */ + @Override + public Vector getChildren() { + if (super.getChildCount() == 0) { + return null; + } + + Vector ret_vec = new Vector(); + for(int i=0; i partSpec; public Partition partHandle; - public tableSpec(Hive db, CommonTree ast, boolean forceCreatePartition) throws SemanticException { + public tableSpec(Hive db, ASTNode ast, boolean forceCreatePartition) throws SemanticException { assert(ast.getToken().getType() == HiveParser.TOK_TAB); int childIndex = 0; @@ -254,10 +252,10 @@ // get partition metadata if partition specified if (ast.getChildCount() == 2) { childIndex = 1; - CommonTree partspec = (CommonTree) ast.getChild(1); + ASTNode partspec = (ASTNode) ast.getChild(1); partSpec = new LinkedHashMap(); for (int i = 0; i < partspec.getChildCount(); ++i) { - CommonTree partspec_val = (CommonTree) partspec.getChild(i); + ASTNode partspec_val = (ASTNode) partspec.getChild(i); String val = stripQuotes(partspec_val.getChild(1).getText()); partSpec.put(unescapeIdentifier(partspec_val.getChild(0).getText()), val); } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTEventDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTEventDispatcher.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTEventDispatcher.java (working copy) @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import java.util.HashMap; -import java.util.ArrayList; - -import org.antlr.runtime.tree.CommonTree; - -/** - * Implementation of a default ParseTreeEventDispatcher. This dispatcher calls - * the associated ParseTreeEventProcessors in the order in which they were - * registered for the event - * - */ -public class DefaultASTEventDispatcher implements - ASTEventDispatcher { - - /** - * Stores the mapping from the ParseTreeEvent to the list of ParseTreeEventProcessors. - * The later are stored in the order that they were registered. - */ - private HashMap> dispatchMap; - - /** - * Constructs the default event dispatcher - */ - public DefaultASTEventDispatcher() { - dispatchMap = new HashMap>(); - } - - /** - * Dispatches the parse subtree to all the event processors registered for the - * event in the order that they were registered. - * - * @see org.apache.hadoop.hive.ql.parse.ASTEventDispatcher#dispatch(org.apache.hadoop.hive.ql.parse.ASTEvent, org.antlr.runtime.tree.CommonTree) - */ - @Override - public void dispatch(ASTEvent evt, CommonTree pt) { - - ArrayList evtp_l = dispatchMap.get(evt); - if (evtp_l == null) { - return; - } - - for(ASTEventProcessor evt_p: evtp_l) { - // Do the actual dispatch - evt_p.process(pt); - } - } - - /** - * Registers the event processor for the event. - * - * @see org.apache.hadoop.hive.ql.parse.ASTEventDispatcher#register(org.apache.hadoop.hive.ql.parse.ASTEvent, org.apache.hadoop.hive.ql.parse.ASTEventProcessor) - */ - @Override - public void register(ASTEvent evt, ASTEventProcessor evt_p) { - - ArrayList evtp_l = dispatchMap.get(evt); - if (evtp_l == null) { - evtp_l = new ArrayList(); - dispatchMap.put(evt, evtp_l); - } - - evtp_l.add(evt_p); - } -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Rule.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Rule.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Rule.java (working copy) @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import java.util.Stack; -import java.io.Serializable; -import org.apache.hadoop.hive.ql.exec.Operator; - -/** - * Rule interface for Operators - * Used in operator dispatching to dispatch process/visitor functions for operators - */ -public interface Rule { - - /** - * @return the cost of the rule - the lower the cost, the better the rule matches - * @throws SemanticException - */ - public int cost(Stack> stack) throws SemanticException; - - /** - * @return the name of the rule - may be useful for debugging - */ - public String getName(); -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -35,6 +35,12 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; +import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; +import org.apache.hadoop.hive.ql.lib.Dispatcher; +import org.apache.hadoop.hive.ql.lib.GraphWalker; +import org.apache.hadoop.hive.ql.lib.NodeProcessor; +import org.apache.hadoop.hive.ql.lib.Rule; +import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.metadata.*; import org.apache.hadoop.hive.ql.optimizer.Optimizer; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext; @@ -53,6 +59,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.fs.Path; import org.apache.commons.lang.StringUtils; @@ -70,7 +77,7 @@ private List loadTableWork; private List loadFileWork; private QB qb; - private CommonTree ast; + private ASTNode ast; private static class Phase1Ctx { String dest; @@ -119,7 +126,7 @@ } @SuppressWarnings("nls") - public void doPhase1QBExpr(CommonTree ast, QBExpr qbexpr, String id, + public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) throws SemanticException { assert (ast.getToken() != null); @@ -136,14 +143,14 @@ // query 1 assert (ast.getChild(0) != null); QBExpr qbexpr1 = new QBExpr(alias + "-subquery1"); - doPhase1QBExpr((CommonTree) ast.getChild(0), qbexpr1, id + "-subquery1", + doPhase1QBExpr((ASTNode) ast.getChild(0), qbexpr1, id + "-subquery1", alias + "-subquery1"); qbexpr.setQBExpr1(qbexpr1); // query 2 assert (ast.getChild(0) != null); QBExpr qbexpr2 = new QBExpr(alias + "-subquery2"); - doPhase1QBExpr((CommonTree) ast.getChild(1), qbexpr2, id + "-subquery2", + doPhase1QBExpr((ASTNode) ast.getChild(1), qbexpr2, id + "-subquery2", alias + "-subquery2"); qbexpr.setQBExpr2(qbexpr2); } @@ -151,13 +158,13 @@ } } - private HashMap doPhase1GetAggregationsFromSelect( - CommonTree selExpr) { + private HashMap doPhase1GetAggregationsFromSelect( + ASTNode selExpr) { // Iterate over the selects search for aggregation Trees. // Use String as keys to eliminate duplicate trees. - HashMap aggregationTrees = new HashMap(); + HashMap aggregationTrees = new HashMap(); for (int i = 0; i < selExpr.getChildCount(); ++i) { - CommonTree sel = (CommonTree) selExpr.getChild(i).getChild(0); + ASTNode sel = (ASTNode) selExpr.getChild(i).getChild(0); doPhase1GetAllAggregations(sel, aggregationTrees); } return aggregationTrees; @@ -172,8 +179,8 @@ * the key to the HashTable is the toStringTree() representation of * the aggregation subtree. */ - private void doPhase1GetAllAggregations(CommonTree expressionTree, - HashMap aggregations) { + private void doPhase1GetAllAggregations(ASTNode expressionTree, + HashMap aggregations) { if (expressionTree.getToken().getType() == HiveParser.TOK_FUNCTION || expressionTree.getToken().getType() == HiveParser.TOK_FUNCTIONDI) { assert (expressionTree.getChildCount() != 0); @@ -185,16 +192,16 @@ } } for (int i = 0; i < expressionTree.getChildCount(); i++) { - doPhase1GetAllAggregations((CommonTree) expressionTree.getChild(i), + doPhase1GetAllAggregations((ASTNode) expressionTree.getChild(i), aggregations); } } - private CommonTree doPhase1GetDistinctFuncExpr( - HashMap aggregationTrees) throws SemanticException { - CommonTree expr = null; - for (Map.Entry entry : aggregationTrees.entrySet()) { - CommonTree value = entry.getValue(); + private ASTNode doPhase1GetDistinctFuncExpr( + HashMap aggregationTrees) throws SemanticException { + ASTNode expr = null; + for (Map.Entry entry : aggregationTrees.entrySet()) { + ASTNode value = entry.getValue(); assert (value != null); if (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI) { if (expr == null) { @@ -207,7 +214,7 @@ return expr; } - private void processTable(QB qb, CommonTree tabref) throws SemanticException { + private void processTable(QB qb, ASTNode tabref) throws SemanticException { // For each table reference get the table name // and the alias (if alias is not present, the table name // is used as an alias) @@ -217,7 +224,7 @@ // tablename tablesample // OR // tablename alias - CommonTree ct = (CommonTree)tabref.getChild(1); + ASTNode ct = (ASTNode)tabref.getChild(1); if (ct.getToken().getType() == HiveParser.TOK_TABLESAMPLE) { tableSamplePresent = true; } @@ -230,18 +237,18 @@ aliasIndex = 2; tableSamplePresent = true; } - CommonTree tableTree = (CommonTree)(tabref.getChild(0)); + ASTNode tableTree = (ASTNode)(tabref.getChild(0)); String alias = unescapeIdentifier(tabref.getChild(aliasIndex).getText()); // If the alias is already there then we have a conflict if (qb.exists(alias)) { throw new SemanticException(ErrorMsg.AMBIGOUS_TABLE_ALIAS.getMsg(tabref.getChild(aliasIndex))); } if (tableSamplePresent) { - CommonTree sampleClause = (CommonTree)tabref.getChild(1); - ArrayList sampleCols = new ArrayList(); + ASTNode sampleClause = (ASTNode)tabref.getChild(1); + ArrayList sampleCols = new ArrayList(); if (sampleClause.getChildCount() > 2) { for (int i = 2; i < sampleClause.getChildCount(); i++) { - sampleCols.add((CommonTree)sampleClause.getChild(i)); + sampleCols.add((ASTNode)sampleClause.getChild(i)); } } // TODO: For now only support sampling on up to two columns @@ -262,13 +269,13 @@ qb.getParseInfo().setSrcForAlias(alias, tableTree); } - private void processSubQuery(QB qb, CommonTree subq) throws SemanticException { + private void processSubQuery(QB qb, ASTNode subq) throws SemanticException { // This is a subquery and must have an alias if (subq.getChildCount() != 2) { throw new SemanticException(ErrorMsg.NO_SUBQUERY_ALIAS.getMsg(subq)); } - CommonTree subqref = (CommonTree) subq.getChild(0); + ASTNode subqref = (ASTNode) subq.getChild(0); String alias = unescapeIdentifier(subq.getChild(1).getText()); // Recursively do the first phase of semantic analysis for the subquery @@ -284,7 +291,7 @@ qb.setSubqAlias(alias, qbexpr); } - private boolean isJoinToken(CommonTree node) + private boolean isJoinToken(ASTNode node) { if ((node.getToken().getType() == HiveParser.TOK_JOIN) || (node.getToken().getType() == HiveParser.TOK_LEFTOUTERJOIN) || @@ -296,13 +303,13 @@ } @SuppressWarnings("nls") - private void processJoin(QB qb, CommonTree join) throws SemanticException { + private void processJoin(QB qb, ASTNode join) throws SemanticException { int numChildren = join.getChildCount(); if ((numChildren != 2) && (numChildren != 3)) throw new SemanticException("Join with multiple children"); for (int num = 0; num < numChildren; num++) { - CommonTree child = (CommonTree) join.getChild(num); + ASTNode child = (ASTNode) join.getChild(num); if (child.getToken().getType() == HiveParser.TOK_TABREF) processTable(qb, child); else if (child.getToken().getType() == HiveParser.TOK_SUBQUERY) @@ -313,7 +320,7 @@ } @SuppressWarnings({"fallthrough", "nls"}) - public void doPhase1(CommonTree ast, QB qb, Phase1Ctx ctx_1) + public void doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) throws SemanticException { QBParseInfo qbp = qb.getParseInfo(); @@ -328,7 +335,7 @@ case HiveParser.TOK_SELECT: qb.countSel(); qbp.setSelExprForClause(ctx_1.dest, ast); - HashMap aggregations = doPhase1GetAggregationsFromSelect(ast); + HashMap aggregations = doPhase1GetAggregationsFromSelect(ast); qbp.setAggregationExprsForClause(ctx_1.dest, aggregations); qbp.setDistinctFuncExprForClause(ctx_1.dest, doPhase1GetDistinctFuncExpr(aggregations)); @@ -345,13 +352,13 @@ // is there a insert in the subquery if (qbp.getIsSubQ()) { - CommonTree ch = (CommonTree)ast.getChild(0); + ASTNode ch = (ASTNode)ast.getChild(0); if ((ch.getToken().getType() != HiveParser.TOK_DIR) || - (((CommonTree)ch.getChild(0)).getToken().getType() != HiveParser.TOK_TMP_FILE)) + (((ASTNode)ch.getChild(0)).getToken().getType() != HiveParser.TOK_TMP_FILE)) throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY.getMsg(ast)); } - qbp.setDestForClause(ctx_1.dest, (CommonTree) ast.getChild(0)); + qbp.setDestForClause(ctx_1.dest, (ASTNode) ast.getChild(0)); } break; @@ -361,7 +368,7 @@ throw new SemanticException("Multiple Children " + child_count); // Check if this is a subquery - CommonTree frm = (CommonTree) ast.getChild(0); + ASTNode frm = (ASTNode) ast.getChild(0); if (frm.getToken().getType() == HiveParser.TOK_TABREF) processTable(qb, frm); else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) @@ -436,7 +443,7 @@ for (int child_pos = 0; child_pos < child_count; ++child_pos) { // Recurse - doPhase1((CommonTree) ast.getChild(child_pos), qb, ctx_1); + doPhase1((ASTNode) ast.getChild(child_pos), qb, ctx_1); } } } @@ -477,9 +484,9 @@ // Pass each where clause to the pruner for(String clause: qbp.getClauseNames()) { - CommonTree whexp = (CommonTree)qbp.getWhrForClause(clause); + ASTNode whexp = (ASTNode)qbp.getWhrForClause(clause); if (whexp != null) { - pruner.addExpression((CommonTree)whexp.getChild(0)); + pruner.addExpression((ASTNode)whexp.getChild(0)); } } @@ -498,15 +505,15 @@ pos++; continue; } - Vector filters = qb.getQbJoinTree().getFilters().get(pos); - for (CommonTree cond : filters) { + Vector filters = qb.getQbJoinTree().getFilters().get(pos); + for (ASTNode cond : filters) { pruner.addJoinOnExpression(cond); if (pruner.hasPartitionPredicate(cond)) joinPartnPruner.put(alias_id, new Boolean(true)); } if (qb.getQbJoinTree().getJoinSrc() != null) { filters = qb.getQbJoinTree().getFilters().get(0); - for (CommonTree cond : filters) { + for (ASTNode cond : filters) { pruner.addJoinOnExpression(cond); if (pruner.hasPartitionPredicate(cond)) joinPartnPruner.put(alias_id, new Boolean(true)); @@ -524,10 +531,10 @@ // Pass each where clause to the pruner for(String clause: qbp.getClauseNames()) { - CommonTree whexp = (CommonTree)qbp.getWhrForClause(clause); + ASTNode whexp = (ASTNode)qbp.getWhrForClause(clause); if (pruner.getTable().isPartitioned() && conf.getVar(HiveConf.ConfVars.HIVEPARTITIONPRUNER).equalsIgnoreCase("strict") && - (whexp == null || !pruner.hasPartitionPredicate((CommonTree)whexp.getChild(0)))) { + (whexp == null || !pruner.hasPartitionPredicate((ASTNode)whexp.getChild(0)))) { throw new SemanticException(ErrorMsg.NO_PARTITION_PREDICATE.getMsg(whexp != null ? whexp : qbp.getSelForClause(clause), " for Alias " + alias + " Table " + pruner.getTable().getName())); } @@ -605,7 +612,7 @@ QBParseInfo qbp = qb.getParseInfo(); for (String name : qbp.getClauseNamesForDest()) { - CommonTree ast = qbp.getDestForClause(name); + ASTNode ast = qbp.getDestForClause(name); switch (ast.getToken().getType()) { case HiveParser.TOK_TAB: { tableSpec ts = new tableSpec(this.db, ast, true); @@ -625,7 +632,7 @@ // This is a dfs file String fname = stripQuotes(ast.getChild(0).getText()); if ((!qb.getParseInfo().getIsSubQ()) && - (((CommonTree)ast.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) + (((ASTNode)ast.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) { fname = getTmpFileName(); ctx.setResDir(new Path(fname)); @@ -689,7 +696,7 @@ @SuppressWarnings("nls") private void parseJoinCondPopulateAlias(QBJoinTree joinTree, - CommonTree condn, Vector leftAliases, Vector rightAliases) + ASTNode condn, Vector leftAliases, Vector rightAliases) throws SemanticException { // String[] allAliases = joinTree.getAllAliases(); switch (condn.getToken().getType()) { @@ -715,19 +722,19 @@ case HiveParser.TOK_FUNCTION: // check all the arguments for (int i = 1; i < condn.getChildCount(); i++) - parseJoinCondPopulateAlias(joinTree, (CommonTree) condn.getChild(i), + parseJoinCondPopulateAlias(joinTree, (ASTNode) condn.getChild(i), leftAliases, rightAliases); break; default: // This is an operator - so check whether it is unary or binary operator if (condn.getChildCount() == 1) - parseJoinCondPopulateAlias(joinTree, (CommonTree) condn.getChild(0), + parseJoinCondPopulateAlias(joinTree, (ASTNode) condn.getChild(0), leftAliases, rightAliases); else if (condn.getChildCount() == 2) { - parseJoinCondPopulateAlias(joinTree, (CommonTree) condn.getChild(0), + parseJoinCondPopulateAlias(joinTree, (ASTNode) condn.getChild(0), leftAliases, rightAliases); - parseJoinCondPopulateAlias(joinTree, (CommonTree) condn.getChild(1), + parseJoinCondPopulateAlias(joinTree, (ASTNode) condn.getChild(1), leftAliases, rightAliases); } else throw new SemanticException(condn.toStringTree() + " encountered with " @@ -737,7 +744,7 @@ } private void populateAliases(Vector leftAliases, - Vector rightAliases, CommonTree condn, QBJoinTree joinTree, + Vector rightAliases, ASTNode condn, QBJoinTree joinTree, Vector leftSrc) throws SemanticException { if ((leftAliases.size() != 0) && (rightAliases.size() != 0)) throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1.getMsg(condn)); @@ -767,7 +774,7 @@ * @param leftSrc left sources * @throws SemanticException */ - private void parseJoinCondition(QBJoinTree joinTree, CommonTree joinCond, Vector leftSrc) + private void parseJoinCondition(QBJoinTree joinTree, ASTNode joinCond, Vector leftSrc) throws SemanticException { if (joinCond == null) return; @@ -777,19 +784,19 @@ throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_3.getMsg(joinCond)); case HiveParser.KW_AND: - parseJoinCondition(joinTree, (CommonTree) joinCond + parseJoinCondition(joinTree, (ASTNode) joinCond .getChild(0), leftSrc); - parseJoinCondition(joinTree, (CommonTree) joinCond + parseJoinCondition(joinTree, (ASTNode) joinCond .getChild(1), leftSrc); break; case HiveParser.EQUAL: - CommonTree leftCondn = (CommonTree) joinCond.getChild(0); + ASTNode leftCondn = (ASTNode) joinCond.getChild(0); Vector leftCondAl1 = new Vector(); Vector leftCondAl2 = new Vector(); parseJoinCondPopulateAlias(joinTree, leftCondn, leftCondAl1, leftCondAl2); - CommonTree rightCondn = (CommonTree) joinCond.getChild(1); + ASTNode rightCondn = (ASTNode) joinCond.getChild(1); Vector rightCondAl1 = new Vector(); Vector rightCondAl2 = new Vector(); parseJoinCondPopulateAlias(joinTree, rightCondn, rightCondAl1, rightCondAl2); @@ -837,7 +844,7 @@ } for (int ci=childrenBegin; ci left : leftAlias) { @@ -879,13 +886,13 @@ private Operator genFilterPlan(String dest, QB qb, Operator input) throws SemanticException { - CommonTree whereExpr = qb.getParseInfo().getWhrForClause(dest); + ASTNode whereExpr = qb.getParseInfo().getWhrForClause(dest); OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRR(); Operator output = putOpInsertMap( OperatorFactory.getAndMakeChild( - new filterDesc(genExprNodeDesc(qb.getMetaData(), (CommonTree)whereExpr.getChild(0), inputRR)), + new filterDesc(genExprNodeDesc(qb.getMetaData(), (ASTNode)whereExpr.getChild(0), inputRR)), new RowSchema(inputRR.getColumnInfos()), input), inputRR); LOG.debug("Created Filter Plan for " + qb.getId() + ":" + dest + " row schema: " + inputRR.toString()); @@ -899,7 +906,7 @@ * @param input the input operator */ @SuppressWarnings("nls") - private Operator genFilterPlan(QB qb, CommonTree condn, Operator input) throws SemanticException { + private Operator genFilterPlan(QB qb, ASTNode condn, Operator input) throws SemanticException { OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRR(); @@ -913,7 +920,7 @@ } @SuppressWarnings("nls") - private void genColList(String tabAlias, String alias, CommonTree sel, + private void genColList(String tabAlias, String alias, ASTNode sel, ArrayList col_list, RowResolver input, Integer pos, RowResolver output) throws SemanticException { @@ -974,7 +981,7 @@ @SuppressWarnings("nls") - private Operator genScriptPlan(CommonTree trfm, QB qb, + private Operator genScriptPlan(ASTNode trfm, QB qb, Operator input) throws SemanticException { // If there is no "AS" clause, the output schema will be "key,value" ArrayList outputColList = new ArrayList(); @@ -983,10 +990,10 @@ outputColList.add("key"); outputColList.add("value"); } else { - CommonTree collist = (CommonTree) trfm.getChild(2); + ASTNode collist = (ASTNode) trfm.getChild(2); int ccount = collist.getChildCount(); for (int i=0; i < ccount; ++i) { - outputColList.add(unescapeIdentifier(((CommonTree)collist.getChild(i)).getText())); + outputColList.add(unescapeIdentifier(((ASTNode)collist.getChild(i)).getText())); } } @@ -1020,26 +1027,26 @@ * This function is a wrapper of parseInfo.getGroupByForClause which automatically * translates SELECT DISTINCT a,b,c to SELECT a,b,c GROUP BY a,b,c. */ - static List getGroupByForClause(QBParseInfo parseInfo, String dest) { + static List getGroupByForClause(QBParseInfo parseInfo, String dest) { if (parseInfo.getSelForClause(dest).getToken().getType() == HiveParser.TOK_SELECTDI) { - CommonTree selectExprs = parseInfo.getSelForClause(dest); - List result = new ArrayList(selectExprs == null + ASTNode selectExprs = parseInfo.getSelForClause(dest); + List result = new ArrayList(selectExprs == null ? 0 : selectExprs.getChildCount()); if (selectExprs != null) { for (int i = 0; i < selectExprs.getChildCount(); ++i) { // table.column AS alias - CommonTree grpbyExpr = (CommonTree) selectExprs.getChild(i).getChild(0); + ASTNode grpbyExpr = (ASTNode) selectExprs.getChild(i).getChild(0); result.add(grpbyExpr); } } return result; } else { - CommonTree grpByExprs = parseInfo.getGroupByForClause(dest); - List result = new ArrayList(grpByExprs == null + ASTNode grpByExprs = parseInfo.getGroupByForClause(dest); + List result = new ArrayList(grpByExprs == null ? 0 : grpByExprs.getChildCount()); if (grpByExprs != null) { for (int i = 0; i < grpByExprs.getChildCount(); ++i) { - CommonTree grpbyExpr = (CommonTree) grpByExprs.getChild(i); + ASTNode grpbyExpr = (ASTNode) grpByExprs.getChild(i); result.add(grpbyExpr); } } @@ -1047,20 +1054,20 @@ } } - private static String getColAlias(CommonTree selExpr, String defaultName) { + private static String getColAlias(ASTNode selExpr, String defaultName) { if (selExpr.getChildCount() == 2) { // return zz for "xx + yy AS zz" return unescapeIdentifier(selExpr.getChild(1).getText()); } - CommonTree root = (CommonTree)selExpr.getChild(0); + ASTNode root = (ASTNode)selExpr.getChild(0); while (root.getType() == HiveParser.DOT || root.getType() == HiveParser.TOK_COLREF) { if (root.getType() == HiveParser.TOK_COLREF && root.getChildCount() == 1) { - root = (CommonTree) root.getChild(0); + root = (ASTNode) root.getChild(0); } else { assert(root.getChildCount() == 2); - root = (CommonTree) root.getChild(1); + root = (ASTNode) root.getChild(1); } } if (root.getType() == HiveParser.Identifier) { @@ -1076,11 +1083,11 @@ private Operator genSelectPlan(String dest, QB qb, Operator input) throws SemanticException { - CommonTree selExprList = qb.getParseInfo().getSelForClause(dest); + ASTNode selExprList = qb.getParseInfo().getSelForClause(dest); ArrayList col_list = new ArrayList(); RowResolver out_rwsch = new RowResolver(); - CommonTree trfm = null; + ASTNode trfm = null; String alias = qb.getParseInfo().getAlias(); Integer pos = Integer.valueOf(0); RowResolver inputRR = opParseCtx.get(input).getRR(); @@ -1090,9 +1097,9 @@ for (int i = 0; i < selExprList.getChildCount(); ++i) { // list of the columns - CommonTree selExpr = (CommonTree) selExprList.getChild(i); + ASTNode selExpr = (ASTNode) selExprList.getChild(i); String colAlias = getColAlias(selExpr, "_C" + i); - CommonTree sel = (CommonTree)selExpr.getChild(0); + ASTNode sel = (ASTNode)selExpr.getChild(0); if (sel.getToken().getType() == HiveParser.TOK_ALLCOLREF) { String tabAlias = null; @@ -1105,9 +1112,9 @@ throw new SemanticException(ErrorMsg.INVALID_TRANSFORM.getMsg(sel)); } trfm = sel; - CommonTree cols = (CommonTree) trfm.getChild(0); + ASTNode cols = (ASTNode) trfm.getChild(0); for (int j = 0; j < cols.getChildCount(); ++j) { - CommonTree expr = (CommonTree) cols.getChild(j); + ASTNode expr = (ASTNode) cols.getChild(j); if (expr.getToken().getType() == HiveParser.TOK_ALLCOLREF) { String tabAlias = null; if (sel.getChildCount() == 1) @@ -1179,17 +1186,17 @@ * @param mode The mode of the aggregation. This affects the evaluate method. * @param aggClasses The classes of the parameters to the UDAF. * @param aggParameters The actual exprNodeDesc of the parameters. - * @param aggTree The CommonTree node of the UDAF in the query. + * @param aggTree The ASTNode node of the UDAF in the query. * @return UDAFInfo * @throws SemanticException when the UDAF is not found or has problems. */ UDAFInfo getUDAFInfo(String aggName, groupByDesc.Mode mode, ArrayList> aggClasses, - ArrayList aggParameters, CommonTree aggTree) throws SemanticException { + ArrayList aggParameters, ASTNode aggTree) throws SemanticException { UDAFInfo r = new UDAFInfo(); r.aggregateMethod = FunctionRegistry.getUDAFMethod(aggName, aggClasses); if (null == r.aggregateMethod) { String reason = "Looking for UDAF \"" + aggName + "\" with parameters " + aggClasses; - throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((CommonTree)aggTree.getChild(0), reason)); + throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)aggTree.getChild(0), reason)); } r.convertedParameters = convertParameters(r.aggregateMethod, aggParameters); @@ -1197,7 +1204,7 @@ r.evaluateMethod = FunctionRegistry.getUDAFEvaluateMethod(aggName, mode); if (r.evaluateMethod == null) { String reason = "UDAF \"" + aggName + "\" does not have evaluate()/evaluatePartial() methods."; - throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((CommonTree)aggTree.getChild(0), reason)); + throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)aggTree.getChild(0), reason)); } return r; @@ -1213,9 +1220,9 @@ groupByOutputRowResolver.setIsExprResolver(true); ArrayList groupByKeys = new ArrayList(); ArrayList aggregations = new ArrayList(); - List grpByExprs = getGroupByForClause(parseInfo, dest); + List grpByExprs = getGroupByForClause(parseInfo, dest); for (int i = 0; i < grpByExprs.size(); ++i) { - CommonTree grpbyExpr = grpByExprs.get(i); + ASTNode grpbyExpr = grpByExprs.get(i); String text = grpbyExpr.toStringTree(); ColumnInfo exprInfo = groupByInputRowResolver.get("",text); @@ -1229,11 +1236,11 @@ new ColumnInfo(field, exprInfo.getType())); } // For each aggregation - HashMap aggregationTrees = parseInfo + HashMap aggregationTrees = parseInfo .getAggregationExprsForClause(dest); assert (aggregationTrees != null); - for (Map.Entry entry : aggregationTrees.entrySet()) { - CommonTree value = entry.getValue(); + for (Map.Entry entry : aggregationTrees.entrySet()) { + ASTNode value = entry.getValue(); String aggName = value.getChild(0).getText(); Class aggClass = FunctionRegistry.getUDAF(aggName); assert (aggClass != null); @@ -1242,7 +1249,7 @@ // 0 is the function name for (int i = 1; i < value.getChildCount(); i++) { String text = value.getChild(i).toStringTree(); - CommonTree paraExpr = (CommonTree)value.getChild(i); + ASTNode paraExpr = (ASTNode)value.getChild(i); ColumnInfo paraExprInfo = groupByInputRowResolver.get("",text); if (paraExprInfo == null) { throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(paraExpr)); @@ -1281,9 +1288,9 @@ groupByOutputRowResolver.setIsExprResolver(true); ArrayList groupByKeys = new ArrayList(); ArrayList aggregations = new ArrayList(); - List grpByExprs = getGroupByForClause(parseInfo, dest); + List grpByExprs = getGroupByForClause(parseInfo, dest); for (int i = 0; i < grpByExprs.size(); ++i) { - CommonTree grpbyExpr = grpByExprs.get(i); + ASTNode grpbyExpr = grpByExprs.get(i); String text = grpbyExpr.toStringTree(); ColumnInfo exprInfo = groupByInputRowResolver.get("",text); @@ -1299,7 +1306,7 @@ // If there is a distinctFuncExp, add all parameters to the reduceKeys. if (parseInfo.getDistinctFuncExprForClause(dest) != null) { - CommonTree value = parseInfo.getDistinctFuncExprForClause(dest); + ASTNode value = parseInfo.getDistinctFuncExprForClause(dest); String aggName = value.getChild(0).getText(); Class aggClass = FunctionRegistry.getUDAF(aggName); assert (aggClass != null); @@ -1308,7 +1315,7 @@ // 0 is the function name for (int i = 1; i < value.getChildCount(); i++) { String text = value.getChild(i).toStringTree(); - CommonTree paraExpr = (CommonTree)value.getChild(i); + ASTNode paraExpr = (ASTNode)value.getChild(i); ColumnInfo paraExprInfo = groupByInputRowResolver.get("",text); if (paraExprInfo == null) { throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(paraExpr)); @@ -1328,10 +1335,10 @@ udaf.evaluateMethod.getReturnType())); } - HashMap aggregationTrees = parseInfo + HashMap aggregationTrees = parseInfo .getAggregationExprsForClause(dest); - for (Map.Entry entry : aggregationTrees.entrySet()) { - CommonTree value = entry.getValue(); + for (Map.Entry entry : aggregationTrees.entrySet()) { + ASTNode value = entry.getValue(); if (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI) continue; @@ -1370,9 +1377,9 @@ groupByOutputRowResolver.setIsExprResolver(true); ArrayList groupByKeys = new ArrayList(); ArrayList aggregations = new ArrayList(); - List grpByExprs = getGroupByForClause(parseInfo, dest); + List grpByExprs = getGroupByForClause(parseInfo, dest); for (int i = 0; i < grpByExprs.size(); ++i) { - CommonTree grpbyExpr = grpByExprs.get(i); + ASTNode grpbyExpr = grpByExprs.get(i); exprNodeDesc grpByExprNode = genExprNodeDesc(qb.getMetaData(), grpbyExpr, groupByInputRowResolver); groupByKeys.add(grpByExprNode); @@ -1383,11 +1390,11 @@ // If there is a distinctFuncExp, add all parameters to the reduceKeys. if (parseInfo.getDistinctFuncExprForClause(dest) != null) { - CommonTree value = parseInfo.getDistinctFuncExprForClause(dest); + ASTNode value = parseInfo.getDistinctFuncExprForClause(dest); int numDistn=0; // 0 is function name for (int i = 1; i < value.getChildCount(); i++) { - CommonTree parameter = (CommonTree) value.getChild(i); + ASTNode parameter = (ASTNode) value.getChild(i); String text = parameter.toStringTree(); if (groupByOutputRowResolver.get("",text) == null) { exprNodeDesc distExprNode = genExprNodeDesc(qb.getMetaData(), parameter, groupByInputRowResolver); @@ -1400,12 +1407,12 @@ } // For each aggregation - HashMap aggregationTrees = parseInfo + HashMap aggregationTrees = parseInfo .getAggregationExprsForClause(dest); assert (aggregationTrees != null); - for (Map.Entry entry : aggregationTrees.entrySet()) { - CommonTree value = entry.getValue(); + for (Map.Entry entry : aggregationTrees.entrySet()) { + ASTNode value = entry.getValue(); String aggName = value.getChild(0).getText(); Class aggClass = FunctionRegistry.getUDAF(aggName); assert (aggClass != null); @@ -1413,7 +1420,7 @@ ArrayList> aggClasses = new ArrayList>(); // 0 is the function name for (int i = 1; i < value.getChildCount(); i++) { - CommonTree paraExpr = (CommonTree)value.getChild(i); + ASTNode paraExpr = (ASTNode)value.getChild(i); exprNodeDesc paraExprNode = genExprNodeDesc(qb.getMetaData(), paraExpr, groupByInputRowResolver); aggParameters.add(paraExprNode); @@ -1482,9 +1489,9 @@ ArrayList reduceKeys = new ArrayList(); // Pre-compute group-by keys and store in reduceKeys - List grpByExprs = getGroupByForClause(parseInfo, dest); + List grpByExprs = getGroupByForClause(parseInfo, dest); for (int i = 0; i < grpByExprs.size(); ++i) { - CommonTree grpbyExpr = grpByExprs.get(i); + ASTNode grpbyExpr = grpByExprs.get(i); String text = grpbyExpr.toStringTree(); if (reduceSinkOutputRowResolver.get("", text) == null) { @@ -1498,10 +1505,10 @@ // If there is a distinctFuncExp, add all parameters to the reduceKeys. if (parseInfo.getDistinctFuncExprForClause(dest) != null) { - CommonTree value = parseInfo.getDistinctFuncExprForClause(dest); + ASTNode value = parseInfo.getDistinctFuncExprForClause(dest); // 0 is function name for (int i = 1; i < value.getChildCount(); i++) { - CommonTree parameter = (CommonTree) value.getChild(i); + ASTNode parameter = (ASTNode) value.getChild(i); String text = parameter.toStringTree(); if (reduceSinkOutputRowResolver.get("",text) == null) { ColumnInfo exprInfo = reduceSinkInputRowResolver.get("", text); @@ -1515,17 +1522,17 @@ // Put partial aggregation results in reduceValues ArrayList reduceValues = new ArrayList(); - HashMap aggregationTrees = parseInfo + HashMap aggregationTrees = parseInfo .getAggregationExprsForClause(dest); int inputField = reduceKeys.size(); - for (Map.Entry entry : aggregationTrees.entrySet()) { + for (Map.Entry entry : aggregationTrees.entrySet()) { TypeInfo type = reduceSinkInputRowResolver.getColumnInfos().get(inputField).getType(); reduceValues.add(new exprNodeColumnDesc( type, (Integer.valueOf(inputField)).toString())); inputField++; - reduceSinkOutputRowResolver.put("", ((CommonTree)entry.getValue()).toStringTree(), + reduceSinkOutputRowResolver.put("", ((ASTNode)entry.getValue()).toStringTree(), new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + (Integer.valueOf(reduceValues.size()-1)).toString(), type)); } @@ -1549,9 +1556,9 @@ ArrayList reduceKeys = new ArrayList(); // Pre-compute group-by keys and store in reduceKeys - List grpByExprs = getGroupByForClause(parseInfo, dest); + List grpByExprs = getGroupByForClause(parseInfo, dest); for (int i = 0; i < grpByExprs.size(); ++i) { - CommonTree grpbyExpr = grpByExprs.get(i); + ASTNode grpbyExpr = grpByExprs.get(i); reduceKeys.add(genExprNodeDesc(qb.getMetaData(), grpbyExpr, reduceSinkInputRowResolver)); String text = grpbyExpr.toStringTree(); if (reduceSinkOutputRowResolver.get("", text) == null) { @@ -1565,10 +1572,10 @@ // If there is a distinctFuncExp, add all parameters to the reduceKeys. if (parseInfo.getDistinctFuncExprForClause(dest) != null) { - CommonTree value = parseInfo.getDistinctFuncExprForClause(dest); + ASTNode value = parseInfo.getDistinctFuncExprForClause(dest); // 0 is function name for (int i = 1; i < value.getChildCount(); i++) { - CommonTree parameter = (CommonTree) value.getChild(i); + ASTNode parameter = (ASTNode) value.getChild(i); String text = parameter.toStringTree(); if (reduceSinkOutputRowResolver.get("",text) == null) { reduceKeys.add(genExprNodeDesc(qb.getMetaData(), parameter, reduceSinkInputRowResolver)); @@ -1581,13 +1588,13 @@ // Put parameters to aggregations in reduceValues ArrayList reduceValues = new ArrayList(); - HashMap aggregationTrees = parseInfo + HashMap aggregationTrees = parseInfo .getAggregationExprsForClause(dest); - for (Map.Entry entry : aggregationTrees.entrySet()) { - CommonTree value = entry.getValue(); + for (Map.Entry entry : aggregationTrees.entrySet()) { + ASTNode value = entry.getValue(); // 0 is function name for (int i = 1; i < value.getChildCount(); i++) { - CommonTree parameter = (CommonTree) value.getChild(i); + ASTNode parameter = (ASTNode) value.getChild(i); String text = parameter.toStringTree(); if (reduceSinkOutputRowResolver.get("",text) == null) { reduceValues.add(genExprNodeDesc(qb.getMetaData(), parameter, reduceSinkInputRowResolver)); @@ -1616,9 +1623,9 @@ reduceSinkOutputRowResolver2.setIsExprResolver(true); ArrayList reduceKeys = new ArrayList(); // Get group-by keys and store in reduceKeys - List grpByExprs = getGroupByForClause(parseInfo, dest); + List grpByExprs = getGroupByForClause(parseInfo, dest); for (int i = 0; i < grpByExprs.size(); ++i) { - CommonTree grpbyExpr = grpByExprs.get(i); + ASTNode grpbyExpr = grpByExprs.get(i); String field = (Integer.valueOf(i)).toString(); TypeInfo typeInfo = reduceSinkInputRowResolver2.get("", grpbyExpr.toStringTree()).getType(); reduceKeys.add(new exprNodeColumnDesc(typeInfo, field)); @@ -1629,11 +1636,11 @@ // Get partial aggregation results and store in reduceValues ArrayList reduceValues = new ArrayList(); int inputField = reduceKeys.size(); - HashMap aggregationTrees = parseInfo + HashMap aggregationTrees = parseInfo .getAggregationExprsForClause(dest); - for (Map.Entry entry : aggregationTrees.entrySet()) { + for (Map.Entry entry : aggregationTrees.entrySet()) { String field = (Integer.valueOf(inputField)).toString(); - CommonTree t = entry.getValue(); + ASTNode t = entry.getValue(); TypeInfo typeInfo = reduceSinkInputRowResolver2.get("", t.toStringTree()).getType(); reduceValues.add(new exprNodeColumnDesc(typeInfo, field)); inputField++; @@ -1660,9 +1667,9 @@ groupByOutputRowResolver2.setIsExprResolver(true); ArrayList groupByKeys = new ArrayList(); ArrayList aggregations = new ArrayList(); - List grpByExprs = getGroupByForClause(parseInfo, dest); + List grpByExprs = getGroupByForClause(parseInfo, dest); for (int i = 0; i < grpByExprs.size(); ++i) { - CommonTree grpbyExpr = grpByExprs.get(i); + ASTNode grpbyExpr = grpByExprs.get(i); String text = grpbyExpr.toStringTree(); ColumnInfo exprInfo = groupByInputRowResolver2.get("",text); if (exprInfo == null) { @@ -1675,10 +1682,10 @@ groupByOutputRowResolver2.put("",grpbyExpr.toStringTree(), new ColumnInfo(field, exprInfo.getType())); } - HashMap aggregationTrees = parseInfo + HashMap aggregationTrees = parseInfo .getAggregationExprsForClause(dest); - for (Map.Entry entry : aggregationTrees.entrySet()) { - CommonTree value = entry.getValue(); + for (Map.Entry entry : aggregationTrees.entrySet()) { + ASTNode value = entry.getValue(); String aggName = value.getChild(0).getText(); Class aggClass = FunctionRegistry.getUDAF(aggName); assert (aggClass != null); @@ -1787,7 +1794,7 @@ } private boolean optimizeMapAggrGroupBy(String dest, QB qb) { - List grpByExprs = getGroupByForClause(qb.getParseInfo(), dest); + List grpByExprs = getGroupByForClause(qb.getParseInfo(), dest); if ((grpByExprs != null) && !grpByExprs.isEmpty()) return false; @@ -2062,7 +2069,7 @@ // First generate the expression for the partition and sort keys // The cluster by clause / distribute by clause has the aliases for partition function - CommonTree partitionExprs = qb.getParseInfo().getClusterByForClause(dest); + ASTNode partitionExprs = qb.getParseInfo().getClusterByForClause(dest); if (partitionExprs == null) { partitionExprs = qb.getParseInfo().getDistributeByForClause(dest); } @@ -2070,7 +2077,7 @@ if (partitionExprs != null) { int ccount = partitionExprs.getChildCount(); for(int i=0; i reduceKeys = new ArrayList(); // Compute join keys and store in reduceKeys - Vector exprs = joinTree.getExpressions().get(pos); + Vector exprs = joinTree.getExpressions().get(pos); for (int i = 0; i < exprs.size(); i++) { - CommonTree expr = exprs.get(i); + ASTNode expr = exprs.get(i); reduceKeys.add(genExprNodeDesc(qb.getMetaData(), expr, inputRS)); } @@ -2247,8 +2254,8 @@ if (leftChild != null) { Operator joinOp = genJoinOperator(qb, leftChild, map); - Vector filter = joinTree.getFilters().get(0); - for (CommonTree cond: filter) + Vector filter = joinTree.getFilters().get(0); + for (ASTNode cond: filter) joinOp = genFilterPlan(qb, cond, joinOp); joinSrcOp = genJoinReduceSinkChild(qb, joinTree, joinOp, null, 0); @@ -2332,7 +2339,7 @@ * traverses the query tree recursively, */ private void pushJoinFilters(QB qb, QBJoinTree joinTree, HashMap map) throws SemanticException { - Vector> filters = joinTree.getFilters(); + Vector> filters = joinTree.getFilters(); if (joinTree.getJoinSrc() != null) pushJoinFilters(qb, joinTree.getJoinSrc(), map); @@ -2340,8 +2347,8 @@ for (String src : joinTree.getBaseSrc()) { if (src != null) { Operator srcOp = map.get(src); - Vector filter = filters.get(pos); - for (CommonTree cond: filter) + Vector filter = filters.get(pos); + for (ASTNode cond: filter) srcOp = genFilterPlan(qb, cond, srcOp); map.put(src, srcOp); } @@ -2349,7 +2356,7 @@ } } - private QBJoinTree genJoinTree(CommonTree joinParseTree) + private QBJoinTree genJoinTree(ASTNode joinParseTree) throws SemanticException { QBJoinTree joinTree = new QBJoinTree(); joinCond[] condn = new joinCond[1]; @@ -2377,8 +2384,8 @@ joinTree.setJoinCond(condn); - CommonTree left = (CommonTree) joinParseTree.getChild(0); - CommonTree right = (CommonTree) joinParseTree.getChild(1); + ASTNode left = (ASTNode) joinParseTree.getChild(0); + ASTNode right = (ASTNode) joinParseTree.getChild(1); if ((left.getToken().getType() == HiveParser.TOK_TABREF) || (left.getToken().getType() == HiveParser.TOK_SUBQUERY)) { @@ -2421,17 +2428,17 @@ } else assert false; - Vector> expressions = new Vector>(); - expressions.add(new Vector()); - expressions.add(new Vector()); + Vector> expressions = new Vector>(); + expressions.add(new Vector()); + expressions.add(new Vector()); joinTree.setExpressions(expressions); - Vector> filters = new Vector>(); - filters.add(new Vector()); - filters.add(new Vector()); + Vector> filters = new Vector>(); + filters.add(new Vector()); + filters.add(new Vector()); joinTree.setFilters(filters); - CommonTree joinCond = (CommonTree) joinParseTree.getChild(2); + ASTNode joinCond = (ASTNode) joinParseTree.getChild(2); Vector leftSrc = new Vector(); parseJoinCondition(joinTree, joinCond, leftSrc); if (leftSrc.size() == 1) @@ -2463,16 +2470,16 @@ baseSrc[i + trgtBaseSrc.length - 1] = nodeBaseSrc[i]; target.setBaseSrc(baseSrc); - Vector> expr = target.getExpressions(); + Vector> expr = target.getExpressions(); for (int i = 0; i < nodeRightAliases.length; i++) expr.add(node.getExpressions().get(i + 1)); - Vector> filter = target.getFilters(); + Vector> filter = target.getFilters(); for (int i = 0; i < nodeRightAliases.length; i++) filter.add(node.getFilters().get(i + 1)); if (node.getFilters().get(0).size() != 0) { - Vector filterPos = filter.get(pos); + Vector filterPos = filter.get(pos); filterPos.addAll(node.getFilters().get(0)); } @@ -2514,8 +2521,8 @@ if (leftAlias == null) return -1; - Vector nodeCondn = node.getExpressions().get(0); - Vector targetCondn = null; + Vector nodeCondn = node.getExpressions().get(0); + Vector targetCondn = null; if (leftAlias.equals(target.getLeftAlias())) { @@ -2727,7 +2734,7 @@ } } else { - for(CommonTree expr: ts.getExprs()) { + for(ASTNode expr: ts.getExprs()) { args.add(genExprNodeDesc(qbm, expr, rwsch)); } } @@ -2799,7 +2806,7 @@ if (ts != null) { int num = ts.getNumerator(); int den = ts.getDenominator(); - ArrayList sampleExprs = ts.getExprs(); + ArrayList sampleExprs = ts.getExprs(); // TODO: Do the type checking of the expressions List tabBucketCols = tab.getBucketCols(); @@ -2831,7 +2838,7 @@ throw new SemanticException(ErrorMsg.TABLE_ALIAS_NOT_ALLOWED.getMsg()); } - if (((CommonTree)sampleExprs.get(i).getChild(0)).getText().equalsIgnoreCase(tabBucketCols.get(j))) { + if (((ASTNode)sampleExprs.get(i).getChild(0)).getText().equalsIgnoreCase(tabBucketCols.get(j))) { colFound = true; } } @@ -2900,7 +2907,7 @@ // process join if (qb.getParseInfo().getJoinExpr() != null) { - CommonTree joinExpr = qb.getParseInfo().getJoinExpr(); + ASTNode joinExpr = qb.getParseInfo().getJoinExpr(); QBJoinTree joinTree = genJoinTree(joinExpr); qb.setQbJoinTree(joinTree); mergeJoinTree(qb); @@ -3029,7 +3036,7 @@ // create a walker which walks the tree in a DFS manner while maintaining the operator stack. The dispatcher // generates the plan from the operator tree - Map opRules = new LinkedHashMap(); + Map opRules = new LinkedHashMap(); opRules.put(new RuleRegExp(new String("R1"), "TS"), new GenMRTableScan1()); opRules.put(new RuleRegExp(new String("R2"), "TS.*RS"), new GenMRRedSink1()); opRules.put(new RuleRegExp(new String("R3"), "RS.*RS"), new GenMRRedSink2()); @@ -3038,8 +3045,10 @@ // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(new GenMROperator(), opRules, procCtx); - OpGraphWalker ogw = new GenMapRedWalker(disp); - ogw.startWalking(this.topOps.values()); + GraphWalker ogw = new GenMapRedWalker(disp); + ArrayList topNodes = new ArrayList(); + topNodes.addAll(this.topOps.values()); + ogw.startWalking(topNodes); // reduce sink does not have any kids breakOperatorTree(procCtx.getRootOps()); @@ -3074,7 +3083,7 @@ @Override @SuppressWarnings("nls") - public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException { + public void analyzeInternal(ASTNode ast, Context ctx) throws SemanticException { this.ctx = ctx; reset(); @@ -3226,7 +3235,7 @@ * @throws SemanticException */ @SuppressWarnings("nls") - private exprNodeDesc genExprNodeDesc(QBMetaData qbm, CommonTree expr, RowResolver input) + private exprNodeDesc genExprNodeDesc(QBMetaData qbm, ASTNode expr, RowResolver input) throws SemanticException { // We recursively create the exprNodeDesc. Base cases: when we encounter // a column ref, we convert that into an exprNodeColumnDesc; when we encounter @@ -3288,7 +3297,7 @@ int childrenBegin = (isFunction ? 1 : 0); ArrayList children = new ArrayList(expr.getChildCount() - childrenBegin); for (int ci=childrenBegin; ci children) { + public static boolean isRedundantConversionFunction(ASTNode expr, boolean isFunction, ArrayList children) { if (!isFunction) return false; // children is always one less than the expr.getChildCount(), since the latter contains function name. assert(children.size() == expr.getChildCount() - 1); // conversion functions take a single parameter if (children.size() != 1) return false; - String funcText = conversionFunctionTextHashMap.get(((CommonTree)expr.getChild(0)).getType()); + String funcText = conversionFunctionTextHashMap.get(((ASTNode)expr.getChild(0)).getType()); // not a conversion function if (funcText == null) return false; // return true when the child type and the conversion target type is the same return children.get(0).getTypeInfo().getPrimitiveClass().getName().equals(funcText); } - public static String getFunctionText(CommonTree expr, boolean isFunction) { + public static String getFunctionText(ASTNode expr, boolean isFunction) { String funcText = null; if (!isFunction) { // For operator, the function name is the operator text, unless it's in our special dictionary @@ -3349,19 +3358,19 @@ // For TOK_FUNCTION, the function name is stored in the first child, unless it's in our // special dictionary. assert(expr.getChildCount() >= 1); - int funcType = ((CommonTree)expr.getChild(0)).getType(); + int funcType = ((ASTNode)expr.getChild(0)).getType(); funcText = specialFunctionTextHashMap.get(funcType); if (funcText == null) { funcText = conversionFunctionTextHashMap.get(funcType); } if (funcText == null) { - funcText = ((CommonTree)expr.getChild(0)).getText(); + funcText = ((ASTNode)expr.getChild(0)).getText(); } } return funcText; } - static exprNodeDesc getXpathOrFuncExprNodeDesc(CommonTree expr, boolean isFunction, + static exprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, boolean isFunction, ArrayList children) throws SemanticException { // return the child directly if the conversion is redundant. @@ -3440,9 +3449,9 @@ Class udf = FunctionRegistry.getUDFClass(funcText); if (udf == null) { if (isFunction) - throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((CommonTree)expr.getChild(0))); + throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)expr.getChild(0))); else - throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((CommonTree)expr)); + throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)expr)); } desc = getFuncExprNodeDesc(funcText, children); @@ -3454,7 +3463,7 @@ if (isFunction) { String reason = "Looking for UDF \"" + expr.getChild(0).getText() + "\" with parameters " + argumentClasses; - throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((CommonTree)expr.getChild(0), reason)); + throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)expr.getChild(0), reason)); } else { String reason = "Looking for Operator \"" + expr.getText() + "\" with parameters " + argumentClasses; throw new SemanticException(ErrorMsg.INVALID_OPERATOR_SIGNATURE.getMsg(expr, reason)); @@ -3475,7 +3484,7 @@ return desc; } - static exprNodeDesc genSimpleExprNodeDesc(CommonTree expr) throws SemanticException { + static exprNodeDesc genSimpleExprNodeDesc(ASTNode expr) throws SemanticException { exprNodeDesc desc = null; switch(expr.getType()) { case HiveParser.TOK_NULL: @@ -3527,7 +3536,7 @@ * @return String * @throws SemanticException */ - static String getTabAliasForCol(QBMetaData qbm, String colName, CommonTree pt) + static String getTabAliasForCol(QBMetaData qbm, String colName, ASTNode pt) throws SemanticException { String tabAlias = null; boolean found = false; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java (working copy) @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.plan.loadFileDesc; import org.apache.hadoop.hive.ql.plan.loadTableDesc; -import org.antlr.runtime.tree.CommonTree; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.conf.HiveConf; @@ -41,7 +40,7 @@ public class ParseContext { private QB qb; - private CommonTree ast; + private ASTNode ast; private HashMap aliasToPruner; private HashMap aliasToSamplePruner; private HashMap> topOps; @@ -73,7 +72,7 @@ * @param topSelOps * list of operators for the selects introduced for column pruning */ - public ParseContext(HiveConf conf, QB qb, CommonTree ast, + public ParseContext(HiveConf conf, QB qb, ASTNode ast, HashMap aliasToPruner, HashMap aliasToSamplePruner, HashMap> topOps, @@ -142,7 +141,7 @@ /** * @return the ast */ - public CommonTree getParseTree() { + public ASTNode getParseTree() { return ast; } @@ -150,7 +149,7 @@ * @param ast * the parsetree to set */ - public void setParseTree(CommonTree ast) { + public void setParseTree(ASTNode ast) { this.ast = ast; } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/RuleRegExp.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/RuleRegExp.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/RuleRegExp.java (working copy) @@ -1,76 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import java.util.Stack; -import java.util.Iterator; -import java.util.regex.Pattern; -import java.util.regex.Matcher; -import java.io.Serializable; - -import org.apache.hadoop.hive.ql.exec.Operator; - -/** - * Rule interface for Operators - * Used in operator dispatching to dispatch process/visitor functions for operators - */ -public class RuleRegExp implements Rule { - - private String ruleName; - private String regExp; - private Pattern pattern; - - /** - * The rule specified by the regular expression. Note that, the regular expression is specified in terms of operator - * name. For eg: TS.*RS -> means TableScan operator followed by anything any number of times followed by ReduceSink - * @param ruleName name of the rule - * @param regExp regular expression for the rule - **/ - public RuleRegExp(String ruleName, String regExp) { - this.ruleName = ruleName; - this.regExp = regExp; - pattern = Pattern.compile(regExp); - } - - /** - * This function returns the cost of the rule for the specified stack. Lower the cost, the better the rule is matched - * @param stacl operator stack encountered so far - * @return cost of the function - * @throws SemanticException - */ - public int cost(Stack> stack) throws SemanticException { - int numElems = stack.size(); - String name = new String(); - for (int pos = numElems - 1; pos >= 0; pos--) { - name = stack.get(pos).getOperatorName().concat(name); - Matcher m = pattern.matcher(name); - if (m.matches()) - return m.group().length(); - } - - return -1; - } - - /** - * @return the name of the operator - **/ - public String getName() { - return ruleName; - } -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultRuleDispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultRuleDispatcher.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultRuleDispatcher.java (working copy) @@ -1,132 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import java.io.Serializable; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.ClassNotFoundException; -import java.util.List; -import java.util.Map; -import java.util.Stack; -import java.lang.ClassNotFoundException; - -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext; -import org.apache.hadoop.hive.ql.exec.OperatorFactory; -import org.apache.hadoop.hive.ql.exec.OperatorFactory.opTuple; - -/** - * Dispatches calls to relevant method in processor. The user registers various rules with the dispatcher, and - * the processor corresponding to closest matching rule is fired. - */ -public class DefaultRuleDispatcher implements Dispatcher { - - private Map opProcRules; - private OperatorProcessorContext opProcCtx; - private OperatorProcessor defaultProc; - - /** - * constructor - * @param defaultProc defualt processor to be fired if no rule matches - * @param opp operator processor that handles actual processing of the node - * @param opProcCtx operator processor context, which is opaque to the dispatcher - */ - public DefaultRuleDispatcher(OperatorProcessor defaultProc, - Map opp, OperatorProcessorContext opProcCtx) { - this.defaultProc = defaultProc; - this.opProcRules = opp; - this.opProcCtx = opProcCtx; - } - - /** - * dispatcher function - * @param op operator to process - * @param opStack the operators encountered so far - * @throws SemanticException - */ - public void dispatch(Operator op, Stack> opStack) - throws SemanticException { - - // find the firing rule - // find the rule from the stack specified - Rule rule = null; - int minCost = Integer.MAX_VALUE; - for (Rule r : opProcRules.keySet()) { - int cost = r.cost(opStack); - if ((cost >= 0) && (cost <= minCost)) { - minCost = cost; - rule = r; - } - } - - OperatorProcessor proc; - - if (rule == null) - proc = defaultProc; - else - proc = opProcRules.get(rule); - - // If the processor has registered a process method for the particular operator, invoke it. - // Otherwise implement the generic function, which would definitely be implemented - for(opTuple opt : OperatorFactory.opvec) { - if(opt.opClass.isInstance(op)) { - Method pcall; - try { - pcall = proc.getClass().getMethod("process", opt.opClass, - Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext")); - pcall.invoke(proc, op, opProcCtx); - return; - } catch (SecurityException e) { - assert false; - } catch (NoSuchMethodException e) { - assert false; - } catch (IllegalArgumentException e) { - assert false; - } catch (IllegalAccessException e) { - assert false; - } catch (InvocationTargetException e) { - throw new SemanticException(e.getTargetException()); - } catch (ClassNotFoundException e) { - assert false; - } - } - } - - try { - // no method found - invoke the generic function - Method pcall = proc.getClass().getMethod("process", Class.forName("org.apache.hadoop.hive.ql.exec.Operator"), - Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext")); - pcall.invoke(proc, ((Operator)op), opProcCtx); - - } catch (SecurityException e) { - assert false; - } catch (NoSuchMethodException e) { - assert false; - } catch (IllegalArgumentException e) { - assert false; - } catch (IllegalAccessException e) { - assert false; - } catch (InvocationTargetException e) { - throw new SemanticException(e.getTargetException()); - } catch (ClassNotFoundException e) { - assert false; - } - } -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java (working copy) @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import java.io.Serializable; -import java.util.Stack; - -import org.apache.hadoop.hive.ql.exec.Operator; - -/** - * Dispatcher interface for Operators - * Used in operator graph walking to dispatch process/visitor functions for operators - */ -public interface Dispatcher { - - /** - * dispatcher function - * @param op operator to process - * @param Stack operator stack to process - * @throws SemanticException - */ - public abstract void dispatch(Operator op, Stack> stack) - throws SemanticException; -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java (working copy) @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import org.antlr.runtime.tree.CommonTree; - -/** - * Interface that a parse tree processor needs to implement - */ -public interface ASTProcessor { - - /** - * Sets the event dispatcher for the processors - * - * @param dispatcher The parse tree event dispatcher - */ - void setDispatcher(ASTEventDispatcher dispatcher); - - /** - * Processes the parse tree and calls the registered event processors - * for the associated parse tree events - * - * @param pt The parse tree to process - */ - void process(CommonTree pt); -} - Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java (working copy) @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - -import org.antlr.runtime.tree.CommonTree; - -/** - * Interface that a parse tree event processor needs to implement. Classes implementing - * this interface and registered with the ParseTreeProcessor are called by the later - * when associated ParseTreeEvents are processed - * - */ -public interface ASTEventProcessor { - - /** - * Processes the parse subtree corresponding to the event - * - * @param pt The parse subtree to process - */ - public void process(CommonTree pt); -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/TableSample.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/TableSample.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TableSample.java (working copy) @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.parse; import java.util.ArrayList; -import org.antlr.runtime.tree.CommonTree; /** * @@ -50,7 +49,7 @@ * the table does not have any clustering column, the usage of a table sample clause * without an ON part is disallowed by the compiler */ - private ArrayList exprs; + private ArrayList exprs; /** * Flag to indicate that input files can be pruned @@ -65,7 +64,7 @@ * @param den The denominator * @param exprs The list of expressions in the ON part of the TABLESAMPLE clause */ - public TableSample(String num, String den, ArrayList exprs) { + public TableSample(String num, String den, ArrayList exprs) { this.numerator = Integer.valueOf(num).intValue(); this.denominator = Integer.valueOf(den).intValue(); this.exprs = exprs; @@ -110,9 +109,9 @@ /** * Gets the ON part's expression list * - * @return ArrayList + * @return ArrayList */ - public ArrayList getExprs() { + public ArrayList getExprs() { return this.exprs; } @@ -121,7 +120,7 @@ * * @param exprs The expression list */ - public void setExprs(ArrayList exprs) { + public void setExprs(ArrayList exprs) { this.exprs = exprs; } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java (working copy) @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.List; -import org.antlr.runtime.tree.CommonTree; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; @@ -37,11 +36,11 @@ super(conf); } - public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException { + public void analyzeInternal(ASTNode ast, Context ctx) throws SemanticException { // Create a semantic analyzer for the query - BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, (CommonTree)ast.getChild(0)); - sem.analyze((CommonTree)ast.getChild(0), ctx); + BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, (ASTNode)ast.getChild(0)); + sem.analyze((ASTNode)ast.getChild(0), ctx); boolean extended = false; if (ast.getChildCount() > 1) { @@ -61,7 +60,7 @@ tasks.add(fetchTask); rootTasks.add(TaskFactory.get(new explainWork(ctx.getResFile(), tasks, - ((CommonTree)ast.getChild(0)).toStringTree(), + ((ASTNode)ast.getChild(0)).toStringTree(), extended), this.conf)); } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java (working copy) @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.ql.parse; -import org.antlr.runtime.tree.CommonTree; - /** * Library of utility functions used in the parse code * @@ -32,7 +30,7 @@ * @param node The parse tree node * @return boolean */ - public static boolean isJoinToken(CommonTree node) { + public static boolean isJoinToken(ASTNode node) { if ((node.getToken().getType() == HiveParser.TOK_JOIN) || (node.getToken().getType() == HiveParser.TOK_LEFTOUTERJOIN) || (node.getToken().getType() == HiveParser.TOK_RIGHTOUTERJOIN) Index: ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java (working copy) @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.parse; - -import java.io.Serializable; - -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext; - -/** - * Base class for processing operators which is no-op. The specific processors can register their own context with - * the dispatcher. - */ -public interface OperatorProcessor { - - /** - * generic process for all ops that don't have specific implementations - * @param op operator to process - * @param opProcCtx operator processor context - * @throws SemanticException - */ - public void process(Operator op, OperatorProcessorContext opProcCtx) - throws SemanticException; -} Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (working copy) @@ -25,8 +25,6 @@ import java.util.List; import java.util.Map; -import org.antlr.runtime.tree.CommonTree; -import org.antlr.runtime.tree.Tree; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -74,7 +72,7 @@ } @Override - public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException { + public void analyzeInternal(ASTNode ast, Context ctx) throws SemanticException { this.ctx = ctx; if (ast.getToken().getType() == HiveParser.TOK_CREATETABLE) analyzeCreateTable(ast, false); @@ -116,7 +114,7 @@ } } - private void analyzeCreateTable(CommonTree ast, boolean isExt) + private void analyzeCreateTable(ASTNode ast, boolean isExt) throws SemanticException { String tableName = unescapeIdentifier(ast.getChild(0).getText()); List cols = null; @@ -139,7 +137,7 @@ int numCh = ast.getChildCount(); for (int num = 1; num < numCh; num++) { - CommonTree child = (CommonTree)ast.getChild(num); + ASTNode child = (ASTNode)ast.getChild(num); switch (child.getToken().getType()) { case HiveParser.TOK_TABCOLLIST: cols = getColumns(child); @@ -148,15 +146,15 @@ comment = unescapeSQLString(child.getChild(0).getText()); break; case HiveParser.TOK_TABLEPARTCOLS: - partCols = getColumns((CommonTree)child.getChild(0)); + partCols = getColumns((ASTNode)child.getChild(0)); break; case HiveParser.TOK_TABLEBUCKETS: - bucketCols = getColumnNames((CommonTree)child.getChild(0)); + bucketCols = getColumnNames((ASTNode)child.getChild(0)); if (child.getChildCount() == 2) numBuckets = (Integer.valueOf(child.getChild(1).getText())).intValue(); else { - sortCols = getColumnNamesOrder((CommonTree)child.getChild(1)); + sortCols = getColumnNamesOrder((ASTNode)child.getChild(1)); numBuckets = (Integer.valueOf(child.getChild(2).getText())).intValue(); } break; @@ -164,7 +162,7 @@ int numChildRowFormat = child.getChildCount(); for (int numC = 0; numC < numChildRowFormat; numC++) { - CommonTree rowChild = (CommonTree)child.getChild(numC); + ASTNode rowChild = (ASTNode)child.getChild(numC); switch (rowChild.getToken().getType()) { case HiveParser.TOK_TABLEROWFORMATFIELD: fieldDelim = unescapeSQLString(rowChild.getChild(0).getText()); @@ -186,7 +184,7 @@ serde = unescapeSQLString(child.getChild(0).getText()); if (child.getChildCount() == 2) { mapProp = new HashMap(); - CommonTree prop = (CommonTree)((CommonTree)child.getChild(1)).getChild(0); + ASTNode prop = (ASTNode)((ASTNode)child.getChild(1)).getChild(0); for (int propChild = 0; propChild < prop.getChildCount(); propChild++) { String key = unescapeSQLString(prop.getChild(propChild).getChild(0).getText()); String value = unescapeSQLString(prop.getChild(propChild).getChild(1).getText()); @@ -300,37 +298,37 @@ } } - private void analyzeDropTable(CommonTree ast) + private void analyzeDropTable(ASTNode ast) throws SemanticException { String tableName = unescapeIdentifier(ast.getChild(0).getText()); dropTableDesc dropTblDesc = new dropTableDesc(tableName); rootTasks.add(TaskFactory.get(new DDLWork(dropTblDesc), conf)); } - private void analyzeAlterTableProps(CommonTree ast) throws SemanticException { + private void analyzeAlterTableProps(ASTNode ast) throws SemanticException { String tableName = unescapeIdentifier(ast.getChild(0).getText()); - HashMap mapProp = getProps((CommonTree)(ast.getChild(1)).getChild(0)); + HashMap mapProp = getProps((ASTNode)(ast.getChild(1)).getChild(0)); alterTableDesc alterTblDesc = new alterTableDesc(alterTableTypes.ADDPROPS); alterTblDesc.setProps(mapProp); alterTblDesc.setOldName(tableName); rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf)); } - private void analyzeAlterTableSerdeProps(CommonTree ast) throws SemanticException { + private void analyzeAlterTableSerdeProps(ASTNode ast) throws SemanticException { String tableName = unescapeIdentifier(ast.getChild(0).getText()); - HashMap mapProp = getProps((CommonTree)(ast.getChild(1)).getChild(0)); + HashMap mapProp = getProps((ASTNode)(ast.getChild(1)).getChild(0)); alterTableDesc alterTblDesc = new alterTableDesc(alterTableTypes.ADDSERDEPROPS); alterTblDesc.setProps(mapProp); alterTblDesc.setOldName(tableName); rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf)); } - private void analyzeAlterTableSerde(CommonTree ast) throws SemanticException { + private void analyzeAlterTableSerde(ASTNode ast) throws SemanticException { String tableName = unescapeIdentifier(ast.getChild(0).getText()); String serdeName = unescapeSQLString(ast.getChild(1).getText()); alterTableDesc alterTblDesc = new alterTableDesc(alterTableTypes.ADDSERDE); if(ast.getChildCount() > 2) { - HashMap mapProp = getProps((CommonTree)(ast.getChild(2)).getChild(0)); + HashMap mapProp = getProps((ASTNode)(ast.getChild(2)).getChild(0)); alterTblDesc.setProps(mapProp); } alterTblDesc.setOldName(tableName); @@ -338,7 +336,7 @@ rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf)); } - private HashMap getProps(CommonTree prop) { + private HashMap getProps(ASTNode prop) { HashMap mapProp = new HashMap(); for (int propChild = 0; propChild < prop.getChildCount(); propChild++) { String key = unescapeSQLString(prop.getChild(propChild).getChild(0).getText()); @@ -348,24 +346,24 @@ return mapProp; } - private List getColumns(CommonTree ast) + private List getColumns(ASTNode ast) { List colList = new ArrayList(); int numCh = ast.getChildCount(); for (int i = 0; i < numCh; i++) { FieldSchema col = new FieldSchema(); - CommonTree child = (CommonTree)ast.getChild(i); + ASTNode child = (ASTNode)ast.getChild(i); col.setName(unescapeIdentifier(child.getChild(0).getText())); - CommonTree typeChild = (CommonTree)(child.getChild(1)); + ASTNode typeChild = (ASTNode)(child.getChild(1)); if (typeChild.getToken().getType() == HiveParser.TOK_LIST) { - CommonTree typName = (CommonTree)typeChild.getChild(0); + ASTNode typName = (ASTNode)typeChild.getChild(0); col.setType(MetaStoreUtils.getListType(getTypeName(typName.getToken().getType()))); } else if (typeChild.getToken().getType() == HiveParser.TOK_MAP) { - CommonTree ltypName = (CommonTree)typeChild.getChild(0); - CommonTree rtypName = (CommonTree)typeChild.getChild(1); + ASTNode ltypName = (ASTNode)typeChild.getChild(0); + ASTNode rtypName = (ASTNode)typeChild.getChild(1); col.setType(MetaStoreUtils.getMapType(getTypeName(ltypName.getToken().getType()), getTypeName(rtypName.getToken().getType()))); } else // primitive type @@ -378,23 +376,23 @@ return colList; } - private List getColumnNames(CommonTree ast) + private List getColumnNames(ASTNode ast) { List colList = new ArrayList(); int numCh = ast.getChildCount(); for (int i = 0; i < numCh; i++) { - CommonTree child = (CommonTree)ast.getChild(i); + ASTNode child = (ASTNode)ast.getChild(i); colList.add(unescapeIdentifier(child.getText())); } return colList; } - private List getColumnNamesOrder(CommonTree ast) + private List getColumnNamesOrder(ASTNode ast) { List colList = new ArrayList(); int numCh = ast.getChildCount(); for (int i = 0; i < numCh; i++) { - CommonTree child = (CommonTree)ast.getChild(i); + ASTNode child = (ASTNode)ast.getChild(i); if (child.getToken().getType() == HiveParser.TOK_TABSORTCOLNAMEASC) colList.add(new Order(unescapeIdentifier(child.getChild(0).getText()), 1)); else @@ -410,29 +408,27 @@ * @param ast The AST from which the qualified name has to be extracted * @return String */ - private String getFullyQualifiedName(CommonTree ast) { + private String getFullyQualifiedName(ASTNode ast) { if (ast.getChildCount() == 0) { return ast.getText(); } - return getFullyQualifiedName((CommonTree)ast.getChild(0)) + "." + - getFullyQualifiedName((CommonTree)ast.getChild(1)); + return getFullyQualifiedName((ASTNode)ast.getChild(0)) + "." + + getFullyQualifiedName((ASTNode)ast.getChild(1)); } - private void analyzeDescribeTable(CommonTree ast) + private void analyzeDescribeTable(ASTNode ast) throws SemanticException { - CommonTree tableTypeExpr = (CommonTree)ast.getChild(0); - // Walk the tree and generate a list of components - ArrayList comp_list = new ArrayList(); - String tableName = getFullyQualifiedName((CommonTree)tableTypeExpr.getChild(0)); + ASTNode tableTypeExpr = (ASTNode)ast.getChild(0); + String tableName = getFullyQualifiedName((ASTNode)tableTypeExpr.getChild(0)); HashMap partSpec = null; // get partition metadata if partition specified if (tableTypeExpr.getChildCount() == 2) { - CommonTree partspec = (CommonTree) tableTypeExpr.getChild(1); + ASTNode partspec = (ASTNode) tableTypeExpr.getChild(1); partSpec = new LinkedHashMap(); for (int i = 0; i < partspec.getChildCount(); ++i) { - CommonTree partspec_val = (CommonTree) partspec.getChild(i); + ASTNode partspec_val = (ASTNode) partspec.getChild(i); String val = stripQuotes(partspec_val.getChild(1).getText()); partSpec.put(partspec_val.getChild(0).getText(), val); } @@ -444,7 +440,7 @@ LOG.info("analyzeDescribeTable done"); } - private void analyzeShowPartitions(CommonTree ast) + private void analyzeShowPartitions(ASTNode ast) throws SemanticException { showPartitionsDesc showPartsDesc; String tableName = unescapeIdentifier(ast.getChild(0).getText()); @@ -452,7 +448,7 @@ rootTasks.add(TaskFactory.get(new DDLWork(showPartsDesc), conf)); } - private void analyzeShowTables(CommonTree ast) + private void analyzeShowTables(ASTNode ast) throws SemanticException { showTablesDesc showTblsDesc; if (ast.getChildCount() == 1) @@ -465,7 +461,7 @@ rootTasks.add(TaskFactory.get(new DDLWork(showTblsDesc), conf)); } - private void analyzeAlterTableRename(CommonTree ast) + private void analyzeAlterTableRename(ASTNode ast) throws SemanticException { alterTableDesc alterTblDesc = new alterTableDesc( unescapeIdentifier(ast.getChild(0).getText()), @@ -473,15 +469,15 @@ rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf)); } - private void analyzeAlterTableModifyCols(CommonTree ast, alterTableTypes alterType) + private void analyzeAlterTableModifyCols(ASTNode ast, alterTableTypes alterType) throws SemanticException { String tblName = unescapeIdentifier(ast.getChild(0).getText()); - List newCols = getColumns((CommonTree)ast.getChild(1)); + List newCols = getColumns((ASTNode)ast.getChild(1)); alterTableDesc alterTblDesc = new alterTableDesc(tblName, newCols, alterType); rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf)); } - private void analyzeAlterTableDropParts(CommonTree ast) throws SemanticException { + private void analyzeAlterTableDropParts(ASTNode ast) throws SemanticException { String tblName = null; List> partSpecs = new ArrayList>(); int childIndex = 0; @@ -489,10 +485,10 @@ tblName = unescapeIdentifier(ast.getChild(0).getText()); // get partition metadata if partition specified for( childIndex = 1; childIndex < ast.getChildCount(); childIndex++) { - CommonTree partspec = (CommonTree) ast.getChild(childIndex); + ASTNode partspec = (ASTNode) ast.getChild(childIndex); HashMap partSpec = new LinkedHashMap(); for (int i = 0; i < partspec.getChildCount(); ++i) { - CommonTree partspec_val = (CommonTree) partspec.getChild(i); + ASTNode partspec_val = (ASTNode) partspec.getChild(i); String val = stripQuotes(partspec_val.getChild(1).getText()); partSpec.put(partspec_val.getChild(0).getText(), val); } Index: ql/src/java/org/apache/hadoop/hive/ql/Driver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java (revision 726865) +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java (working copy) @@ -22,7 +22,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.*; -import org.antlr.runtime.tree.CommonTree; +import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.commons.lang.StringUtils; @@ -160,10 +160,10 @@ resStream = null; pd = new ParseDriver(); - CommonTree tree = pd.parse(command); + ASTNode tree = pd.parse(command); while((tree.getToken() == null) && (tree.getChildCount() > 0)) { - tree = (CommonTree)tree.getChild(0); + tree = (ASTNode)tree.getChild(0); } sem = SemanticAnalyzerFactory.get(conf, tree);