diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java index 3097385..0c111bc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -34,6 +35,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; /** @@ -45,33 +47,27 @@ * */ public final class RewriteParseContextGenerator { - private static final Log LOG = LogFactory.getLog(RewriteParseContextGenerator.class.getName()); - private RewriteParseContextGenerator(){ - } + private static final Log LOG = LogFactory.getLog(RewriteParseContextGenerator.class.getName()); /** - * Parse the input {@link String} command and generate a ASTNode tree. + * Parse the input {@link String} command and generate an operator tree. * @param conf * @param command - * @return the parse context * @throws SemanticException */ - public static ParseContext generateOperatorTree(HiveConf conf, - String command) throws SemanticException{ - Context ctx; - ParseContext subPCtx = null; + public static Operator generateOperatorTree(HiveConf conf, + String command) throws SemanticException { + Operator operatorTree; try { - ctx = new Context(conf); + Context ctx = new Context(conf); ParseDriver pd = new ParseDriver(); ASTNode tree = pd.parse(command, ctx); tree = ParseUtils.findRootNonNullToken(tree); BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, tree); assert(sem instanceof SemanticAnalyzer); - doSemanticAnalysis((SemanticAnalyzer) sem, tree, ctx); - - subPCtx = ((SemanticAnalyzer) sem).getParseContext(); + operatorTree = doSemanticAnalysis((SemanticAnalyzer) sem, tree, ctx); LOG.info("Sub-query Semantic Analysis Completed"); } catch (IOException e) { LOG.error("IOException in generating the operator " + @@ -89,13 +85,12 @@ public static ParseContext generateOperatorTree(HiveConf conf, LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); throw new SemanticException(e.getMessage(), e); } - return subPCtx; - + return operatorTree; } /** * For the input ASTNode tree, perform a semantic analysis and check metadata - * Generate a operator tree and return the {@link ParseContext} instance for the operator tree. + * Generate a operator tree and return it. * * @param ctx * @param sem @@ -103,7 +98,7 @@ public static ParseContext generateOperatorTree(HiveConf conf, * @return * @throws SemanticException */ - private static void doSemanticAnalysis(SemanticAnalyzer sem, + private static Operator doSemanticAnalysis(SemanticAnalyzer sem, ASTNode ast, Context ctx) throws SemanticException { QB qb = new QB(null, null, false); ASTNode child = ast; @@ -119,9 +114,10 @@ private static void doSemanticAnalysis(SemanticAnalyzer sem, LOG.info("Completed getting MetaData in Sub-query Semantic Analysis"); LOG.info("Sub-query Abstract syntax tree: " + ast.toStringTree()); - sem.genPlan(qb); + Operator operator = sem.genPlan(qb); LOG.info("Sub-query Completed plan generation"); + return operator; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java index 69a5a44..8544ca8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java @@ -23,7 +23,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,6 +31,7 @@ import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorUtils; import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; @@ -267,15 +267,14 @@ private void replaceGroupByOperatorProcess(GroupByOperator operator, int index) String selReplacementCommand = "select sum(`" + rewriteQueryCtx.getAggregateFunction() + "`)" + " from " + rewriteQueryCtx.getIndexName() + " group by " + rewriteQueryCtx.getIndexKey() + " "; - // create a new ParseContext for the query to retrieve its operator tree, - // and the required GroupByOperator from it - ParseContext newDAGContext = RewriteParseContextGenerator.generateOperatorTree( - rewriteQueryCtx.getParseContext().getConf(), selReplacementCommand); + // retrieve the operator tree for the query, and the required GroupByOperator from it + Operator newOperatorTree = RewriteParseContextGenerator.generateOperatorTree( + rewriteQueryCtx.getParseContext().getConf(), + selReplacementCommand); // we get our new GroupByOperator here - Map> newGbyOpMap = newDAGContext.getGroupOpToInputTables(); - GroupByOperator newGbyOperator = newGbyOpMap.keySet().iterator().next(); - GroupByDesc oldConf = operator.getConf(); + GroupByOperator newGbyOperator = + OperatorUtils.findSingleOperatorUpstream(newOperatorTree, GroupByOperator.class); // we need this information to set the correct colList, outputColumnNames // in SelectOperator @@ -297,6 +296,7 @@ private void replaceGroupByOperatorProcess(GroupByOperator operator, int index) // Now the GroupByOperator has the new AggregationList; // sum(`_count_of_indexed_key`) // instead of count(indexed_key) + GroupByDesc oldConf = operator.getConf(); oldConf.setAggregators((ArrayList) newAggrList); operator.setConf(oldConf); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java index dda4f75..e061b6c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; import org.apache.hadoop.hive.ql.exec.FetchTask; -import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.ListSinkOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; @@ -84,7 +83,6 @@ private List> listMapJoinOpsNoReducer; // list of map join // operators with no // reducer - private Map> groupOpToInputTables; private Map prunedPartitions; private Map viewAliasToInput; @@ -155,7 +153,6 @@ public ParseContext( List loadTableWork, List loadFileWork, Context ctx, HashMap idToTableNameMap, int destTableId, UnionProcContext uCtx, List> listMapJoinOpsNoReducer, - Map> groupOpToInputTables, Map prunedPartitions, HashMap opToSamplePruner, GlobalLimitCtx globalLimitCtx, @@ -180,7 +177,6 @@ public ParseContext( this.destTableId = destTableId; this.uCtx = uCtx; this.listMapJoinOpsNoReducer = listMapJoinOpsNoReducer; - this.groupOpToInputTables = groupOpToInputTables; this.prunedPartitions = prunedPartitions; this.opToSamplePruner = opToSamplePruner; this.nameToSplitSample = nameToSplitSample; @@ -409,21 +405,6 @@ public void setOpToSamplePruner( } /** - * @return the groupOpToInputTables - */ - public Map> getGroupOpToInputTables() { - return groupOpToInputTables; - } - - /** - * @param groupOpToInputTables - */ - public void setGroupOpToInputTables( - Map> groupOpToInputTables) { - this.groupOpToInputTables = groupOpToInputTables; - } - - /** * @return pruned partition map */ public Map getPrunedPartitions() { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index c9a5ce5..d1a89ad 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -389,7 +389,6 @@ public void initParseCtx(ParseContext pctx) { uCtx = pctx.getUCtx(); listMapJoinOpsNoReducer = pctx.getListMapJoinOpsNoReducer(); qb = pctx.getQB(); - groupOpToInputTables = pctx.getGroupOpToInputTables(); prunedPartitions = pctx.getPrunedPartitions(); fetchTask = pctx.getFetchTask(); setLineageInfo(pctx.getLineageInfo()); @@ -400,7 +399,7 @@ public ParseContext getParseContext() { new HashSet(joinContext.keySet()), new HashSet(smbMapJoinContext.keySet()), loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, - listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, + listMapJoinOpsNoReducer, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties); @@ -10163,7 +10162,7 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce new HashSet(joinContext.keySet()), new HashSet(smbMapJoinContext.keySet()), loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, - listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, opToSamplePruner, + listMapJoinOpsNoReducer, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 0116c85..78998f3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -390,7 +390,7 @@ public ParseContext getParseContext(ParseContext pCtx, List