diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java index 911ac8a..7fa0dd6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java @@ -44,7 +44,7 @@ private final HashMap aliasToSrc; private final HashMap nameToDest; private final HashMap nameToSample; - private final Map exprToColumnAlias; + private final Map> destToAliases; private final Map destToSelExpr; private final HashMap destToWhereExpr; private final HashMap destToGroupby; @@ -111,7 +111,7 @@ public QBParseInfo(String alias, boolean isSubQ) { aliasToSrc = new HashMap(); nameToDest = new HashMap(); nameToSample = new HashMap(); - exprToColumnAlias = new HashMap(); + destToAliases = new HashMap>(); destToLateralView = new HashMap(); destToSelExpr = new LinkedHashMap(); destToWhereExpr = new HashMap(); @@ -401,20 +401,16 @@ public void setTabSample(String alias, TableSample tableSample) { nameToSample.put(alias.toLowerCase(), tableSample); } - public String getExprToColumnAlias(ASTNode expr) { - return exprToColumnAlias.get(expr); + public Map getAllExprToColumnAlias(String dest) { + return destToAliases.get(dest); } - public Map getAllExprToColumnAlias() { - return exprToColumnAlias; - } - - public boolean hasExprToColumnAlias(ASTNode expr) { - return exprToColumnAlias.containsKey(expr); - } - - public void setExprToColumnAlias(ASTNode expr, String alias) { - exprToColumnAlias.put(expr, alias); + public void setExprToColumnAlias(String dest, String alias, ASTNode expr) { + Map map = destToAliases.get(dest); + if (map == null) { + destToAliases.put(dest, map = new HashMap()); + } + map.put(alias, expr); } public void setDestLimit(String dest, Integer limit) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index e4a30a2..8a54fca 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -439,14 +439,14 @@ public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) return aggregationTrees; } - private void doPhase1GetColumnAliasesFromSelect( + private void doPhase1GetColumnAliasesFromSelect(String clause, ASTNode selectExpr, QBParseInfo qbp) { for (int i = 0; i < selectExpr.getChildCount(); ++i) { ASTNode selExpr = (ASTNode) selectExpr.getChild(i); if ((selExpr.getToken().getType() == HiveParser.TOK_SELEXPR) && (selExpr.getChildCount() == 2)) { String columnAlias = unescapeIdentifier(selExpr.getChild(1).getText()); - qbp.setExprToColumnAlias((ASTNode) selExpr.getChild(0), columnAlias); + qbp.setExprToColumnAlias(clause, columnAlias, (ASTNode) selExpr.getChild(0)); } } } @@ -929,7 +929,7 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) LinkedHashMap aggregations = doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest); - doPhase1GetColumnAliasesFromSelect(ast, qbp); + doPhase1GetColumnAliasesFromSelect(ctx_1.dest, ast, qbp); qbp.setAggregationExprsForClause(ctx_1.dest, aggregations); qbp.setDistinctFuncExprsForClause(ctx_1.dest, doPhase1GetDistinctFuncExprs(aggregations)); @@ -2125,10 +2125,15 @@ private Operator genHavingPlan(String dest, QB qb, Operator input, OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); - Map exprToColumnAlias = qb.getParseInfo().getAllExprToColumnAlias(); - for (ASTNode astNode : exprToColumnAlias.keySet()) { - if (inputRR.getExpression(astNode) != null) { - inputRR.put("", exprToColumnAlias.get(astNode), inputRR.getExpression(astNode)); + Map exprToColumnAlias = qb.getParseInfo().getAllExprToColumnAlias(dest); + + Map resolved = new HashMap(); + if (exprToColumnAlias != null) { + for (Map.Entry entry : exprToColumnAlias.entrySet()) { + ExprNodeDesc exprNodeDesc = genExprNodeDesc(entry.getValue(), inputRR); + if (exprNodeDesc != null) { + resolved.put(entry.getKey(), exprNodeDesc); + } } } ASTNode condn = (ASTNode) havingExpr.getChild(0); @@ -2138,7 +2143,7 @@ private Operator genHavingPlan(String dest, QB qb, Operator input, * so we invoke genFilterPlan to handle SubQuery algebraic transformation, * just as is done for SubQuery predicates appearing in the Where Clause. */ - Operator output = genFilterPlan(condn, qb, input, aliasToOpInfo, true); + Operator output = genFilterPlan(condn, qb, input, aliasToOpInfo, resolved); output = putOpInsertMap(output, inputRR); return output; } @@ -2157,12 +2162,14 @@ private Operator genPlanForSubQueryPredicate( @SuppressWarnings("nls") private Operator genFilterPlan(ASTNode searchCond, QB qb, Operator input, Map aliasToOpInfo, - boolean forHavingClause) + Map resolved) throws SemanticException { OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); + boolean forHavingClause = resolved != null; + /* * Handling of SubQuery Expressions: * if "Where clause contains no SubQuery expressions" then @@ -2299,7 +2306,7 @@ private Operator genFilterPlan(ASTNode searchCond, QB qb, Operator input, } } - return genFilterPlan(qb, searchCond, input); + return genFilterPlan(qb, searchCond, input, resolved); } /** @@ -2313,13 +2320,13 @@ private Operator genFilterPlan(ASTNode searchCond, QB qb, Operator input, * the input operator */ @SuppressWarnings("nls") - private Operator genFilterPlan(QB qb, ASTNode condn, Operator input) - throws SemanticException { + private Operator genFilterPlan(QB qb, ASTNode condn, Operator input, + Map resolved) throws SemanticException { OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( - new FilterDesc(genExprNodeDesc(condn, inputRR), false), new RowSchema( + new FilterDesc(genExprNodeDesc(condn, inputRR, resolved), false), new RowSchema( inputRR.getColumnInfos()), input), inputRR); if (LOG.isDebugEnabled()) { @@ -4790,7 +4797,7 @@ private Operator genGroupByPlan1ReduceMultiGBY(List dests, QB qb, Operat if (parseInfo.getWhrForClause(dest) != null) { ASTNode whereExpr = qb.getParseInfo().getWhrForClause(dest); - curr = genFilterPlan((ASTNode) whereExpr.getChild(0), qb, forwardOp, aliasToOpInfo, false); + curr = genFilterPlan((ASTNode) whereExpr.getChild(0), qb, forwardOp, aliasToOpInfo, null); } // Generate GroupbyOperator @@ -6896,7 +6903,7 @@ private Operator genJoinOperator(QB qb, QBJoinTree joinTree, if ( joinSrcOp != null ) { ArrayList filter = joinTree.getFiltersForPushing().get(0); for (ASTNode cond : filter) { - joinSrcOp = genFilterPlan(qb, cond, joinSrcOp); + joinSrcOp = genFilterPlan(qb, cond, joinSrcOp, null); } } @@ -6951,7 +6958,7 @@ private Operator genJoinOperator(QB qb, QBJoinTree joinTree, Operator op = joinOp; for(ASTNode condn : joinTree.getPostJoinFilters() ) { - op = genFilterPlan(qb, condn, op); + op = genFilterPlan(qb, condn, op, null); } return op; } @@ -7120,7 +7127,7 @@ private void pushJoinFilters(QB qb, QBJoinTree joinTree, Operator srcOp = map.get(src); ArrayList filter = filters.get(pos); for (ASTNode cond : filter) { - srcOp = genFilterPlan(qb, cond, srcOp); + srcOp = genFilterPlan(qb, cond, srcOp, null); } map.put(src, srcOp); } @@ -8328,7 +8335,7 @@ private Operator genBodyPlan(QB qb, Operator input, Map aliasT if (qbp.getWhrForClause(dest) != null) { ASTNode whereExpr = qb.getParseInfo().getWhrForClause(dest); - curr = genFilterPlan((ASTNode) whereExpr.getChild(0), qb, curr, aliasToOpInfo, false); + curr = genFilterPlan((ASTNode) whereExpr.getChild(0), qb, curr, aliasToOpInfo, null); } if (qbp.getAggregationExprsForClause(dest).size() != 0 @@ -9734,11 +9741,16 @@ private void saveViewDefinition() throws SemanticException { /** * Generates an expression node descriptor for the expression with TypeCheckCtx. */ - public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input) + public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input) throws SemanticException { + return genExprNodeDesc(expr, input, (Map)null); + } + + public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, Map resolved) throws SemanticException { // Since the user didn't supply a customized type-checking context, // use default settings. TypeCheckCtx tcCtx = new TypeCheckCtx(input); + tcCtx.setColMapping(resolved); return genExprNodeDesc(expr, input, tcCtx); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java index a95ae20..89687c1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java @@ -19,6 +19,9 @@ package org.apache.hadoop.hive.ql.parse; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; + +import java.util.Map; /** * This class implements the context information that is used for typechecking @@ -48,6 +51,11 @@ private ASTNode errorSrcNode; /** + * column alias to ExprNodeDesc, which is resolved already + */ + private Map colMapping; + + /** * Whether to allow stateful UDF invocations. */ private boolean allowStatefulFunctions; @@ -139,4 +147,12 @@ public void setAllowDistinctFunctions(boolean allowDistinctFunctions) { public boolean isAllowDistinctFunctions() { return allowDistinctFunctions; } + + public ExprNodeDesc getColMapping(String alias) { + return colMapping == null ? null : colMapping.get(alias); + } + + public void setColMapping(Map colMapping) { + this.colMapping = colMapping; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index e44f5ae..25c1065 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -505,6 +505,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } } else { if (colInfo == null) { + ExprNodeDesc resolved = ctx.getColMapping(tableOrCol); + if (resolved != null) { + return resolved; + } // It's not a column or a table alias. if (input.getIsExprResolver()) { ASTNode exprNode = expr; diff --git ql/src/test/queries/clientpositive/having3.q ql/src/test/queries/clientpositive/having3.q new file mode 100644 index 0000000..c28c080 --- /dev/null +++ ql/src/test/queries/clientpositive/having3.q @@ -0,0 +1,4 @@ +explain +select value,max(key)-min(key) as span from src tablesample (10 rows) group by value having span>=0; + +select value,max(key)-min(key) as span from src tablesample (10 rows) group by value having span>=0; diff --git ql/src/test/results/clientpositive/having3.q.out ql/src/test/results/clientpositive/having3.q.out new file mode 100644 index 0000000..b4515c4 --- /dev/null +++ ql/src/test/results/clientpositive/having3.q.out @@ -0,0 +1,80 @@ +PREHOOK: query: explain +select value,max(key)-min(key) as span from src tablesample (10 rows) group by value having span>=0 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select value,max(key)-min(key) as span from src tablesample (10 rows) group by value having span>=0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Row Limit Per Split: 10 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(key), min(key) + keys: value (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string) + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0), min(VALUE._col1) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col1 - _col2) >= 0) (type: boolean) + Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), (_col1 - _col2) (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select value,max(key)-min(key) as span from src tablesample (10 rows) group by value having span>=0 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select value,max(key)-min(key) as span from src tablesample (10 rows) group by value having span>=0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +val_165 0.0 +val_238 0.0 +val_255 0.0 +val_27 0.0 +val_278 0.0 +val_311 0.0 +val_409 0.0 +val_484 0.0 +val_86 0.0 +val_98 0.0