diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 8e00e0b..f2419ef 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -34,6 +34,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -2371,6 +2372,13 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException // 8. We create the group_by operator gbRel = genGBRelNode(gbExprNDescLst, aggregations, groupingSets, srcRel); + Map exprToColumnAlias = qb.getParseInfo().getAllExprToColumnAlias(); + for (ASTNode astNode : exprToColumnAlias.keySet()) { + if (groupByOutputRowResolver.getExpression(astNode) != null) { + groupByOutputRowResolver.put("", exprToColumnAlias.get(astNode), + groupByOutputRowResolver.getExpression(astNode)); + } + } relToHiveColNameCalcitePosMap.put(gbRel, buildHiveToCalciteColumnMap(groupByOutputRowResolver, gbRel)); this.relToHiveRR.put(gbRel, groupByOutputRowResolver); @@ -2910,6 +2918,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) ASTNode exprList = selExprList; int startPosn = posn; List tabAliasesForAllProjs = getTabAliases(starRR); + Map colNames = new LinkedHashMap<>(); for (int i = startPosn; i < exprList.getChildCount(); ++i) { // 6.1 child can be EXPR AS ALIAS, or EXPR. @@ -2995,18 +3004,20 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) UnsupportedFeature.Duplicates_in_RR); } + // We store the colnames and put without check later. if (exp instanceof ExprNodeColumnDesc) { - ExprNodeColumnDesc colExp = (ExprNodeColumnDesc) exp; - String[] altMapping = inputRR.getAlternateMappings(colExp.getColumn()); - if (altMapping != null) { - // TODO: this can overwrite the mapping. Should this be allowed? - out_rwsch.put(altMapping[0], altMapping[1], colInfo); - } + colNames.put(((ExprNodeColumnDesc) exp).getColumn(), colInfo); } - pos = Integer.valueOf(pos.intValue() + 1); } } + // We put the additional mapping without check. + for (Entry entry : colNames.entrySet()) { + String[] altMapping = inputRR.getAlternateMappings(entry.getKey()); + if (altMapping != null) { + out_rwsch.put(altMapping[0], altMapping[1], entry.getValue()); + } + } selectStar = selectStar && exprList.getChildCount() == posn + 1; // 7. Convert Hive projections to Calcite diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java index 3a226e7..f47833f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java @@ -122,7 +122,7 @@ public QBParseInfo(String alias, boolean isSubQ) { nameToDest = new HashMap(); nameToDestSchema = new HashMap>(); nameToSample = new HashMap(); - exprToColumnAlias = new HashMap(); + exprToColumnAlias = new LinkedHashMap(); destToLateralView = new HashMap(); destToSelExpr = new LinkedHashMap(); destToWhereExpr = new HashMap(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 06db7f9..5e38fef 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2944,27 +2944,14 @@ private void extractJoinCondsFromWhereClause(QBJoinTree joinTree, QB qb, String private Operator genHavingPlan(String dest, QB qb, Operator input, Map aliasToOpInfo) throws SemanticException { - ASTNode havingExpr = qb.getParseInfo().getHavingForClause(dest); - - OpParseContext inputCtx = opParseCtx.get(input); - RowResolver inputRR = inputCtx.getRowResolver(); - Map exprToColumnAlias = qb.getParseInfo().getAllExprToColumnAlias(); - for (ASTNode astNode : exprToColumnAlias.keySet()) { - if (inputRR.getExpression(astNode) != null) { - inputRR.put("", exprToColumnAlias.get(astNode), inputRR.getExpression(astNode)); - } - } ASTNode condn = (ASTNode) havingExpr.getChild(0); - /* * Now a having clause can contain a SubQuery predicate; * so we invoke genFilterPlan to handle SubQuery algebraic transformation, * just as is done for SubQuery predicates appearing in the Where Clause. */ - Operator output = genFilterPlan(condn, qb, input, aliasToOpInfo, true, false); - output = putOpInsertMap(output, inputRR); - return output; + return genFilterPlan(condn, qb, input, aliasToOpInfo, true, false); } private Operator genPlanForSubQueryPredicate( @@ -4164,14 +4151,6 @@ static boolean isRegex(String pattern, HiveConf conf) { .isSkewedCol() : false); out_rwsch.put(tabAlias, colAlias, colInfo); - if ( exp instanceof ExprNodeColumnDesc ) { - ExprNodeColumnDesc colExp = (ExprNodeColumnDesc) exp; - String[] altMapping = inputRR.getAlternateMappings(colExp.getColumn()); - if ( altMapping != null ) { - out_rwsch.put(altMapping[0], altMapping[1], colInfo); - } - } - pos = Integer.valueOf(pos.intValue() + 1); } } @@ -9265,6 +9244,15 @@ private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb, throws SemanticException { QBParseInfo qbp = qb.getParseInfo(); + + OpParseContext inputCtx = opParseCtx.get(curr); + RowResolver inputRR = inputCtx.getRowResolver(); + Map exprToColumnAlias = qb.getParseInfo().getAllExprToColumnAlias(); + for (ASTNode astNode : exprToColumnAlias.keySet()) { + if (inputRR.getExpression(astNode) != null) { + inputRR.put("", exprToColumnAlias.get(astNode), inputRR.getExpression(astNode)); + } + } // Insert HAVING plan here if (qbp.getHavingForClause(dest) != null) { @@ -9272,6 +9260,7 @@ private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb, throw new SemanticException("HAVING specified without GROUP BY"); } curr = genHavingPlan(dest, qb, curr, aliasToOpInfo); + curr = putOpInsertMap(curr, inputRR); } diff --git a/ql/src/test/queries/clientnegative/alias_groupby_orderby_1.q b/ql/src/test/queries/clientnegative/alias_groupby_orderby_1.q new file mode 100644 index 0000000..002e48c --- /dev/null +++ b/ql/src/test/queries/clientnegative/alias_groupby_orderby_1.q @@ -0,0 +1,2 @@ +set hive.cbo.enable=false; +select key as c0, count(*) as c1 from src group by key order by key; diff --git a/ql/src/test/queries/clientnegative/alias_groupby_orderby_2.q b/ql/src/test/queries/clientnegative/alias_groupby_orderby_2.q new file mode 100644 index 0000000..a011773 --- /dev/null +++ b/ql/src/test/queries/clientnegative/alias_groupby_orderby_2.q @@ -0,0 +1,2 @@ +set hive.cbo.enable=false; +select key as c0, count(*) as c1 from src group by c0; diff --git a/ql/src/test/queries/clientnegative/cbo_alias_groupby_orderby_1.q b/ql/src/test/queries/clientnegative/cbo_alias_groupby_orderby_1.q new file mode 100644 index 0000000..dc864b0 --- /dev/null +++ b/ql/src/test/queries/clientnegative/cbo_alias_groupby_orderby_1.q @@ -0,0 +1 @@ +select key as c0, count(*) as c1 from src group by key order by key; diff --git a/ql/src/test/queries/clientnegative/cbo_alias_groupby_orderby_2.q b/ql/src/test/queries/clientnegative/cbo_alias_groupby_orderby_2.q new file mode 100644 index 0000000..c2d36e8 --- /dev/null +++ b/ql/src/test/queries/clientnegative/cbo_alias_groupby_orderby_2.q @@ -0,0 +1 @@ +select key as c0, count(*) as c1 from src group by c0; diff --git a/ql/src/test/queries/clientpositive/lineage3.q b/ql/src/test/queries/clientpositive/lineage3.q index c907e21..2b92d0a 100644 --- a/ql/src/test/queries/clientpositive/lineage3.q +++ b/ql/src/test/queries/clientpositive/lineage3.q @@ -87,7 +87,7 @@ from ( from alltypesorc where cint is not null group by cint, ctinyint - order by cint, ctinyint + order by c1, c2 limit 5 ) x ) x2 @@ -141,7 +141,7 @@ from ( select cint c1, ctinyint c2, min(cfloat) c3 from alltypesorc group by cint, ctinyint - order by cint, ctinyint + order by c1, c2 limit 1 ) x ) x2 diff --git a/ql/src/test/results/clientnegative/alias_groupby_orderby_1.q.out b/ql/src/test/results/clientnegative/alias_groupby_orderby_1.q.out new file mode 100644 index 0000000..66bfefa --- /dev/null +++ b/ql/src/test/results/clientnegative/alias_groupby_orderby_1.q.out @@ -0,0 +1 @@ +FAILED: SemanticException [Error 10004]: Line 2:64 Invalid table alias or column reference 'key': (possible column names are: c0, c1) diff --git a/ql/src/test/results/clientnegative/alias_groupby_orderby_2.q.out b/ql/src/test/results/clientnegative/alias_groupby_orderby_2.q.out new file mode 100644 index 0000000..d0bbfe3 --- /dev/null +++ b/ql/src/test/results/clientnegative/alias_groupby_orderby_2.q.out @@ -0,0 +1 @@ +FAILED: SemanticException [Error 10004]: Line 2:51 Invalid table alias or column reference 'c0': (possible column names are: key, value) diff --git a/ql/src/test/results/clientnegative/cbo_alias_groupby_orderby_1.q.out b/ql/src/test/results/clientnegative/cbo_alias_groupby_orderby_1.q.out new file mode 100644 index 0000000..c2e5f94 --- /dev/null +++ b/ql/src/test/results/clientnegative/cbo_alias_groupby_orderby_1.q.out @@ -0,0 +1 @@ +FAILED: SemanticException [Error 10004]: Line 1:64 Invalid table alias or column reference 'key': (possible column names are: c0, c1) diff --git a/ql/src/test/results/clientnegative/cbo_alias_groupby_orderby_2.q.out b/ql/src/test/results/clientnegative/cbo_alias_groupby_orderby_2.q.out new file mode 100644 index 0000000..eef29de --- /dev/null +++ b/ql/src/test/results/clientnegative/cbo_alias_groupby_orderby_2.q.out @@ -0,0 +1 @@ +FAILED: SemanticException [Error 10004]: Line 1:51 Invalid table alias or column reference 'c0': (possible column names are: key, value) diff --git a/ql/src/test/results/clientpositive/lineage3.q.out b/ql/src/test/results/clientpositive/lineage3.q.out index 61acf52..2e66fe5 100644 --- a/ql/src/test/results/clientpositive/lineage3.q.out +++ b/ql/src/test/results/clientpositive/lineage3.q.out @@ -147,7 +147,7 @@ from ( from alltypesorc where cint is not null group by cint, ctinyint - order by cint, ctinyint + order by c1, c2 limit 5 ) x ) x2 @@ -158,7 +158,7 @@ order by x2, c1 desc PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -{"version":"1.0","engine":"mr","database":"default","hash":"bc64f8bec21631969a17930ec609cde9","queryText":"select c1, x2, x3\nfrom (\n select c1, min(c2) x2, sum(c3) x3\n from (\n select c1, c2, c3\n from (\n select cint c1, ctinyint c2, min(cbigint) c3\n from alltypesorc\n where cint is not null\n group by cint, ctinyint\n order by cint, ctinyint\n limit 5\n ) x\n ) x2\n group by c1\n) y\nwhere x2 > 0\norder by x2, c1 desc","edges":[{"sources":[3],"targets":[0],"edgeType":"PROJECTION"},{"sources":[4],"targets":[1],"expression":"min(default.alltypesorc.ctinyint)","edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"expression":"sum(min(default.alltypesorc.cbigint))","edgeType":"PROJECTION"},{"sources":[3],"targets":[0,1,2],"expression":"alltypesorc.cint is not null","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1,2],"expression":"(min(default.alltypesorc.ctinyint) > 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c1"},{"id":1,"vertexType":"COLUMN","vertexId":"x2"},{"id":2,"vertexType":"COLUMN","vertexId":"x3"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"4d4f1f731ce2fb3e0b466ed6108ffe04","queryText":"select c1, x2, x3\nfrom (\n select c1, min(c2) x2, sum(c3) x3\n from (\n select c1, c2, c3\n from (\n select cint c1, ctinyint c2, min(cbigint) c3\n from alltypesorc\n where cint is not null\n group by cint, ctinyint\n order by c1, c2\n limit 5\n ) x\n ) x2\n group by c1\n) y\nwhere x2 > 0\norder by x2, c1 desc","edges":[{"sources":[3],"targets":[0],"edgeType":"PROJECTION"},{"sources":[4],"targets":[1],"expression":"min(default.alltypesorc.ctinyint)","edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"expression":"sum(min(default.alltypesorc.cbigint))","edgeType":"PROJECTION"},{"sources":[3],"targets":[0,1,2],"expression":"alltypesorc.cint is not null","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1,2],"expression":"(min(default.alltypesorc.ctinyint) > 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c1"},{"id":1,"vertexType":"COLUMN","vertexId":"x2"},{"id":2,"vertexType":"COLUMN","vertexId":"x3"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]} -1072910839 11 2048385991 -1073279343 11 -1595604468 PREHOOK: query: select key, value from src1 @@ -265,7 +265,7 @@ from ( select cint c1, ctinyint c2, min(cfloat) c3 from alltypesorc group by cint, ctinyint - order by cint, ctinyint + order by c1, c2 limit 1 ) x ) x2 @@ -276,7 +276,7 @@ PREHOOK: type: CREATEVIEW PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@dest_v2 -{"version":"1.0","engine":"mr","database":"default","hash":"eda442b42b9c3a9cbdb7aff1984ad2dd","queryText":"create view dest_v2 (a, b) as select c1, x2\nfrom (\n select c1, min(c2) x2\n from (\n select c1, c2, c3\n from (\n select cint c1, ctinyint c2, min(cfloat) c3\n from alltypesorc\n group by cint, ctinyint\n order by cint, ctinyint\n limit 1\n ) x\n ) x2\n group by c1\n) y\norder by x2,c1 desc","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"min(default.alltypesorc.ctinyint)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_v2.c1"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_v2.x2"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"2d0ee57b4cfc20815364207cfe499bda","queryText":"create view dest_v2 (a, b) as select c1, x2\nfrom (\n select c1, min(c2) x2\n from (\n select c1, c2, c3\n from (\n select cint c1, ctinyint c2, min(cfloat) c3\n from alltypesorc\n group by cint, ctinyint\n order by c1, c2\n limit 1\n ) x\n ) x2\n group by c1\n) y\norder by x2,c1 desc","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"min(default.alltypesorc.ctinyint)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_v2.c1"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_v2.x2"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"}]} PREHOOK: query: drop view if exists dest_v3 PREHOOK: type: DROPVIEW PREHOOK: query: create view dest_v3 (a1, a2, a3, a4, a5, a6, a7) as