diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java index b56b608..b987a28 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java @@ -75,7 +75,7 @@ public static RewriteCanApplyCtx getInstance(ParseContext parseContext){ private String baseTableName; private String indexTableName; private String aggFunction; - + private TableScanOperator tableScanOperator; private List selectOperators; private List groupByOperators; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java index 3097385..8424445 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ParseContext; @@ -47,20 +48,22 @@ public final class RewriteParseContextGenerator { private static final Log LOG = LogFactory.getLog(RewriteParseContextGenerator.class.getName()); - private RewriteParseContextGenerator(){ + private Operator operatorTree; + private ParseContext parseContext; + + public RewriteParseContextGenerator(HiveConf conf, String command) throws SemanticException { + generateOperatorTree(conf, command); } /** - * Parse the input {@link String} command and generate a ASTNode tree. + * Parse the input {@link String} command and generate an operator tree. * @param conf * @param command - * @return the parse context * @throws SemanticException */ - public static ParseContext generateOperatorTree(HiveConf conf, + private void generateOperatorTree(HiveConf conf, String command) throws SemanticException{ Context ctx; - ParseContext subPCtx = null; try { ctx = new Context(conf); ParseDriver pd = new ParseDriver(); @@ -69,9 +72,8 @@ public static ParseContext generateOperatorTree(HiveConf conf, BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, tree); assert(sem instanceof SemanticAnalyzer); - doSemanticAnalysis((SemanticAnalyzer) sem, tree, ctx); - - subPCtx = ((SemanticAnalyzer) sem).getParseContext(); + this.operatorTree = doSemanticAnalysis((SemanticAnalyzer) sem, tree, ctx); + this.parseContext = ((SemanticAnalyzer) sem).getParseContext(); LOG.info("Sub-query Semantic Analysis Completed"); } catch (IOException e) { LOG.error("IOException in generating the operator " + @@ -89,13 +91,11 @@ public static ParseContext generateOperatorTree(HiveConf conf, LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); throw new SemanticException(e.getMessage(), e); } - return subPCtx; - } /** * For the input ASTNode tree, perform a semantic analysis and check metadata - * Generate a operator tree and return the {@link ParseContext} instance for the operator tree. + * Generate a operator tree and return it. * * @param ctx * @param sem @@ -103,7 +103,7 @@ public static ParseContext generateOperatorTree(HiveConf conf, * @return * @throws SemanticException */ - private static void doSemanticAnalysis(SemanticAnalyzer sem, + private static Operator doSemanticAnalysis(SemanticAnalyzer sem, ASTNode ast, Context ctx) throws SemanticException { QB qb = new QB(null, null, false); ASTNode child = ast; @@ -119,9 +119,18 @@ private static void doSemanticAnalysis(SemanticAnalyzer sem, LOG.info("Completed getting MetaData in Sub-query Semantic Analysis"); LOG.info("Sub-query Abstract syntax tree: " + ast.toStringTree()); - sem.genPlan(qb); + Operator operator = sem.genPlan(qb); LOG.info("Sub-query Completed plan generation"); + return operator; + } + + public Operator getOperatorTree() { + return operatorTree; + } + + public ParseContext getParseContext() { + return parseContext; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java index 72f4588..8a7ba53 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java @@ -24,18 +24,18 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorUtils; import org.apache.hadoop.hive.ql.exec.RowSchema; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.SelectOperator; -import org.apache.hadoop.hive.ql.exec.GroupByOperator; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -290,13 +290,14 @@ private void replaceGroupByOperatorProcess(GroupByOperator operator, int index) + rewriteQueryCtx.getIndexKey() + " "; // create a new ParseContext for the query to retrieve its operator tree, // and the required GroupByOperator from it - ParseContext newDAGContext = RewriteParseContextGenerator.generateOperatorTree( + RewriteParseContextGenerator generator = new RewriteParseContextGenerator( rewriteQueryCtx.getParseContext().getConf(), selReplacementCommand); + Operator newOperatorTree = generator.getOperatorTree(); + ParseContext newDAGContext = generator.getParseContext(); // we get our new GroupByOperator here - Map> newGbyOpMap = newDAGContext.getGroupOpToInputTables(); - GroupByOperator newGbyOperator = newGbyOpMap.keySet().iterator().next(); - GroupByDesc oldConf = operator.getConf(); + GroupByOperator newGbyOperator = + OperatorUtils.findSingleOperatorUpstream(newOperatorTree, GroupByOperator.class); // we need this information to set the correct colList, outputColumnNames // in SelectOperator @@ -323,6 +324,7 @@ private void replaceGroupByOperatorProcess(GroupByOperator operator, int index) gbyOPC.setRowResolver(gbyRR); rewriteQueryCtx.getOpc().put(operator, gbyOPC); + GroupByDesc oldConf = operator.getConf(); oldConf.setAggregators((ArrayList) newAggrList); operator.setConf(oldConf); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java index b838bff..0013d41 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java @@ -31,7 +31,6 @@ import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; import org.apache.hadoop.hive.ql.exec.FetchTask; -import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.ListSinkOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; @@ -86,7 +85,6 @@ private List> listMapJoinOpsNoReducer; // list of map join // operators with no // reducer - private Map> groupOpToInputTables; private Map prunedPartitions; private Map viewAliasToInput; @@ -158,7 +156,6 @@ public ParseContext( List loadTableWork, List loadFileWork, Context ctx, HashMap idToTableNameMap, int destTableId, UnionProcContext uCtx, List> listMapJoinOpsNoReducer, - Map> groupOpToInputTables, Map prunedPartitions, HashMap opToSamplePruner, GlobalLimitCtx globalLimitCtx, @@ -184,7 +181,6 @@ public ParseContext( this.destTableId = destTableId; this.uCtx = uCtx; this.listMapJoinOpsNoReducer = listMapJoinOpsNoReducer; - this.groupOpToInputTables = groupOpToInputTables; this.prunedPartitions = prunedPartitions; this.opToSamplePruner = opToSamplePruner; this.nameToSplitSample = nameToSplitSample; @@ -450,21 +446,6 @@ public void setOpToSamplePruner( } /** - * @return the groupOpToInputTables - */ - public Map> getGroupOpToInputTables() { - return groupOpToInputTables; - } - - /** - * @param groupOpToInputTables - */ - public void setGroupOpToInputTables( - Map> groupOpToInputTables) { - this.groupOpToInputTables = groupOpToInputTables; - } - - /** * @return pruned partition map */ public Map getPrunedPartitions() { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 4364f28..a66d4da 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -390,7 +390,6 @@ public void initParseCtx(ParseContext pctx) { uCtx = pctx.getUCtx(); listMapJoinOpsNoReducer = pctx.getListMapJoinOpsNoReducer(); qb = pctx.getQB(); - groupOpToInputTables = pctx.getGroupOpToInputTables(); prunedPartitions = pctx.getPrunedPartitions(); fetchTask = pctx.getFetchTask(); setLineageInfo(pctx.getLineageInfo()); @@ -402,7 +401,7 @@ public ParseContext getParseContext() { new HashSet(joinContext.keySet()), new HashSet(smbMapJoinContext.keySet()), loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, - listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, + listMapJoinOpsNoReducer, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties); @@ -10168,7 +10167,7 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce new HashSet(joinContext.keySet()), new HashSet(smbMapJoinContext.keySet()), loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, - listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, opToSamplePruner, + listMapJoinOpsNoReducer, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index f2eb4d2..15af56f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -390,7 +390,7 @@ public ParseContext getParseContext(ParseContext pCtx, List