diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java index b3c4b47..432322d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java @@ -130,6 +130,10 @@ public static String getAppendedAliasFromId(String outer_id, String alias) { return (outer_id == null ? alias : outer_id + ":" + alias); } + public String getAlias() { + return qbp.getAlias(); + } + public QBParseInfo getParseInfo() { return qbp; } @@ -248,6 +252,12 @@ public boolean getIsQuery() { return isQuery; } + // decide whether to rewrite RR of subquery + public boolean isTopLevelSelectStarQuery() { + return !isCTAS() && qbp.isTopLevelSimpleSelectStarQuery(); + } + + // find target for fetch task conversion optimizer (not allows subqueries) public boolean isSimpleSelectQuery() { return qbp.isSimpleSelectQuery() && aliasToSubq.isEmpty() && !isCTAS() && !qbp.isAnalyzeCommand(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java index 86e4602..02c4be9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; +import org.antlr.runtime.tree.Tree; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec; @@ -449,39 +450,49 @@ public void setOuterQueryLimit(int outerQueryLimit) { this.outerQueryLimit = outerQueryLimit; } + public boolean isTopLevelSimpleSelectStarQuery() { + if (alias != null || destToSelExpr.size() != 1 || !isSimpleSelectQuery()) { + return false; + } + for (ASTNode selExprs : destToSelExpr.values()) { + if (selExprs.getChildCount() != 1) { + return false; + } + Tree sel = selExprs.getChild(0).getChild(0); + if (sel == null || sel.getType() != HiveParser.TOK_ALLCOLREF) { + return false; + } + } + return true; + } + public boolean isSimpleSelectQuery() { - if (isSubQ || (joinExpr != null) - || (!destToGroupby.isEmpty()) || (!destToClusterby.isEmpty()) - || (!aliasToLateralViews.isEmpty())) { + if (isSubQ || joinExpr != null || !destToOrderby.isEmpty() || !destToSortby.isEmpty() + || !destToGroupby.isEmpty() || !destToClusterby.isEmpty() || !destToDistributeby.isEmpty() + || !aliasToLateralViews.isEmpty() || !destToLateralView.isEmpty()) { return false; } - Iterator>> aggrIter = destToAggregationExprs - .entrySet().iterator(); - while (aggrIter.hasNext()) { - HashMap h = aggrIter.next().getValue(); - if ((h != null) && (!h.isEmpty())) { + for (Map entry : destToAggregationExprs.values()) { + if (entry != null && !entry.isEmpty()) { return false; } } - if (!destToDistinctFuncExprs.isEmpty()) { - Iterator>> distn = destToDistinctFuncExprs - .entrySet().iterator(); - while (distn.hasNext()) { - List ct = distn.next().getValue(); - if (!ct.isEmpty()) { - return false; - } + for (Map entry : destToWindowingExprs.values()) { + if (entry != null && !entry.isEmpty()) { + return false; + } + } + + for (List ct : destToDistinctFuncExprs.values()) { + if (!ct.isEmpty()) { + return false; } } - Iterator> iter = nameToDest.entrySet() - .iterator(); - while (iter.hasNext()) { - Map.Entry entry = iter.next(); - ASTNode v = entry.getValue(); - if (!(((ASTNode)v.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) { + for (ASTNode v : nameToDest.values()) { + if (!(v.getChild(0).getType() == HiveParser.TOK_TMP_FILE)) { return false; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java index 9c55379..10ac4b2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java @@ -29,7 +29,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.RowSchema; @@ -195,17 +194,6 @@ public ColumnInfo get(String tab_alias, String col_alias) throws SemanticExcepti return ret; } - /** - * check if column name is already exist in RR - */ - public void checkColumn(String tableAlias, String columnAlias) throws SemanticException { - ColumnInfo prev = get(null, columnAlias); - if (prev != null && - (tableAlias == null || !tableAlias.equalsIgnoreCase(prev.getTabAlias()))) { - throw new SemanticException(ErrorMsg.AMBIGUOUS_COLUMN.getMsg(columnAlias)); - } - } - public ArrayList getColumnInfos() { return rowSchema.getSignature(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 19110ce..00e528e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -37,7 +37,6 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -69,7 +68,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; @@ -2708,7 +2706,7 @@ private Operator genNotNullFilterForJoinSourcePlan(QB qb, Operator input, @SuppressWarnings("nls") private Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, ArrayList col_list, - RowResolver input, Integer pos, RowResolver output, List aliases, boolean subQuery) + RowResolver input, Integer pos, RowResolver output, List aliases) throws SemanticException { // The table alias should exist @@ -2766,9 +2764,6 @@ private Integer genColListRegex(String colRegex, String tabAlias, continue; } - if (subQuery) { - output.checkColumn(tmp[0], tmp[1]); - } ColumnInfo oColInfo = inputColsProcessed.get(colInfo); if (oColInfo == null) { ExprNodeColumnDesc expr = new ExprNodeColumnDesc(colInfo.getType(), @@ -3394,7 +3389,6 @@ private static boolean isRegex(String pattern, HiveConf conf) { posn++; } - boolean subQuery = qb.getParseInfo().getIsSubQ(); boolean isInTransform = (selExprList.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM); if (isInTransform) { @@ -3432,7 +3426,7 @@ private static boolean isRegex(String pattern, HiveConf conf) { } if (isUDTF && (selectStar = udtfExprType == HiveParser.TOK_FUNCTIONSTAR)) { genColListRegex(".*", null, (ASTNode) udtfExpr.getChild(0), - col_list, inputRR, pos, out_rwsch, qb.getAliases(), subQuery); + col_list, inputRR, pos, out_rwsch, qb.getAliases()); } } @@ -3554,7 +3548,7 @@ private static boolean isRegex(String pattern, HiveConf conf) { if (expr.getType() == HiveParser.TOK_ALLCOLREF) { pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), - expr, col_list, inputRR, pos, out_rwsch, qb.getAliases(), subQuery); + expr, col_list, inputRR, pos, out_rwsch, qb.getAliases()); selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause && !inputRR.getIsExprResolver() @@ -3563,7 +3557,7 @@ private static boolean isRegex(String pattern, HiveConf conf) { // This can only happen without AS clause // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), - null, expr, col_list, inputRR, pos, out_rwsch, qb.getAliases(), subQuery); + null, expr, col_list, inputRR, pos, out_rwsch, qb.getAliases()); } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0) @@ -3576,7 +3570,7 @@ private static boolean isRegex(String pattern, HiveConf conf) { pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()), unescapeIdentifier(expr.getChild(0).getChild(0).getText() .toLowerCase()), expr, col_list, inputRR, pos, out_rwsch, - qb.getAliases(), subQuery); + qb.getAliases()); } else { // Case when this is an expression TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); @@ -3590,9 +3584,6 @@ private static boolean isRegex(String pattern, HiveConf conf) { colAlias = recommended; } col_list.add(exp); - if (subQuery) { - out_rwsch.checkColumn(tabAlias, colAlias); - } ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(pos), exp.getWritableObjectInspector(), tabAlias, false); @@ -8906,24 +8897,6 @@ private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb, } } - // change curr ops row resolver's tab aliases to query alias if it - // exists - if (qb.getParseInfo().getAlias() != null) { - RowResolver rr = opParseCtx.get(curr).getRowResolver(); - RowResolver newRR = new RowResolver(); - String alias = qb.getParseInfo().getAlias(); - for (ColumnInfo colInfo : rr.getColumnInfos()) { - String name = colInfo.getInternalName(); - String[] tmp = rr.reverseLookup(name); - if ("".equals(tmp[0]) || tmp[1] == null) { - // ast expression is not a valid column name for table - tmp[1] = colInfo.getInternalName(); - } - newRR.put(alias, tmp[1], colInfo); - } - opParseCtx.get(curr).setRowResolver(newRR); - } - return curr; } @@ -9508,13 +9481,14 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String } } - private Operator genPlan(QBExpr qbexpr) throws SemanticException { + private Operator genPlan(QB parent, QBExpr qbexpr) throws SemanticException { if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { - return genPlan(qbexpr.getQB()); + boolean skipAmbiguityCheck = viewSelect == null && parent.isTopLevelSelectStarQuery(); + return genPlan(qbexpr.getQB(), skipAmbiguityCheck); } if (qbexpr.getOpcode() == QBExpr.Opcode.UNION) { - Operator qbexpr1Ops = genPlan(qbexpr.getQBExpr1()); - Operator qbexpr2Ops = genPlan(qbexpr.getQBExpr2()); + Operator qbexpr1Ops = genPlan(parent, qbexpr.getQBExpr1()); + Operator qbexpr2Ops = genPlan(parent, qbexpr.getQBExpr2()); return genUnionPlan(qbexpr.getAlias(), qbexpr.getQBExpr1().getAlias(), qbexpr1Ops, qbexpr.getQBExpr2().getAlias(), qbexpr2Ops); @@ -9522,8 +9496,13 @@ private Operator genPlan(QBExpr qbexpr) throws SemanticException { return null; } - @SuppressWarnings("nls") public Operator genPlan(QB qb) throws SemanticException { + return genPlan(qb, false); + } + + @SuppressWarnings("nls") + public Operator genPlan(QB qb, boolean skipAmbiguityCheck) + throws SemanticException { // First generate all the opInfos for the elements in the from clause Map aliasToOpInfo = new HashMap(); @@ -9531,8 +9510,7 @@ public Operator genPlan(QB qb) throws SemanticException { // Recurse over the subqueries to fill the subquery part of the plan for (String alias : qb.getSubqAliases()) { QBExpr qbexpr = qb.getSubqForAlias(alias); - aliasToOpInfo.put(alias, genPlan(qbexpr)); - qbexpr.setAlias(alias); + aliasToOpInfo.put(alias, genPlan(qb, qbexpr)); } // Recurse over all the source tables @@ -9631,10 +9609,38 @@ public Operator genPlan(QB qb) throws SemanticException { LOG.debug("Created Plan for Query Block " + qb.getId()); } + if (qb.getAlias() != null) { + rewriteRRForSubQ(qb.getAlias(), bodyOpInfo, skipAmbiguityCheck); + } + this.qb = qb; return bodyOpInfo; } + // change curr ops row resolver's tab aliases to subq alias + private void rewriteRRForSubQ(String alias, Operator operator, boolean skipAmbiguityCheck) + throws SemanticException { + RowResolver rr = opParseCtx.get(operator).getRowResolver(); + RowResolver newRR = new RowResolver(); + for (ColumnInfo colInfo : rr.getColumnInfos()) { + String name = colInfo.getInternalName(); + String[] tmp = rr.reverseLookup(name); + if ("".equals(tmp[0]) || tmp[1] == null) { + // ast expression is not a valid column name for table + tmp[1] = colInfo.getInternalName(); + } else if (newRR.get(alias, tmp[1]) != null) { + // enforce uniqueness of column names + if (!skipAmbiguityCheck) { + throw new SemanticException(ErrorMsg.AMBIGUOUS_COLUMN.getMsg(tmp[1] + " in " + alias)); + } + // if it's wrapped by top-level select star query, skip ambiguity check (for backward compatibility) + tmp[1] = colInfo.getInternalName(); + } + newRR.put(alias, tmp[1], colInfo); + } + opParseCtx.get(operator).setRowResolver(newRR); + } + private Table getDummyTable() throws SemanticException { Path dummyPath = createDummyFile(); Table desc = new Table(DUMMY_DATABASE, DUMMY_TABLE); @@ -13669,7 +13675,6 @@ private RelNode genSelectForWindowing(QB qb, RelNode srcRel) throws SemanticExce ColumnInfo oColInfo = new ColumnInfo( getColumnInternalName(projsForWindowSelOp.size()), wtp.getValue(), null, false); if (false) { - out_rwsch.checkColumn(null, wExprSpec.getAlias()); out_rwsch.put(null, wExprSpec.getAlias(), oColInfo); } else { out_rwsch.putExpression(wExprSpec.getExpression(), oColInfo); @@ -13806,9 +13811,6 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep throw new OptiqSemanticException(msg); } - // 4. Determine if select corresponds to a subquery - subQuery = qb.getParseInfo().getIsSubQ(); - // 4. Bailout if select involves Transform boolean isInTransform = (selExprList.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM); if (isInTransform) { @@ -13871,8 +13873,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep if (expr.getType() == HiveParser.TOK_ALLCOLREF) { pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)) - .toLowerCase(), expr, col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, - subQuery); + .toLowerCase(), expr, col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs); selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause && !inputRR.getIsExprResolver() @@ -13881,7 +13882,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep // This can only happen without AS clause // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), null, expr, - col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, subQuery); + col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs); } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0).getChild(0).getText() @@ -13892,7 +13893,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()), unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()), expr, - col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs, subQuery); + col_list, inputRR, pos, out_rwsch, tabAliasesForAllProjs); } else { // Case when this is an expression TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR); @@ -13904,9 +13905,6 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel) throws SemanticExcep colAlias = recommended; } col_list.add(exp); - if (subQuery) { - out_rwsch.checkColumn(tabAlias, colAlias); - } ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(pos), exp.getWritableObjectInspector(), tabAlias, false); @@ -13982,7 +13980,6 @@ private RelNode genLogicalPlan(QB qb, boolean outerMostQB) throws SemanticExcept for (String subqAlias : qb.getSubqAliases()) { QBExpr qbexpr = qb.getSubqForAlias(subqAlias); aliasToRel.put(subqAlias, genLogicalPlan(qbexpr)); - qbexpr.setAlias(subqAlias); } // 1.2 Recurse over all the source tables diff --git ql/src/test/queries/clientnegative/ambiguous_col0.q ql/src/test/queries/clientnegative/ambiguous_col0.q deleted file mode 100644 index 46349c6..0000000 --- ql/src/test/queries/clientnegative/ambiguous_col0.q +++ /dev/null @@ -1,2 +0,0 @@ --- TOK_ALLCOLREF -explain select * from (select * from (select * from src) a join (select * from src1) b on (a.key = b.key)) t; diff --git ql/src/test/queries/clientnegative/ambiguous_col1.q ql/src/test/queries/clientnegative/ambiguous_col1.q deleted file mode 100644 index 9e8bcbd..0000000 --- ql/src/test/queries/clientnegative/ambiguous_col1.q +++ /dev/null @@ -1,3 +0,0 @@ -set hive.support.quoted.identifiers=none; --- TOK_TABLE_OR_COL -explain select * from (select `.*` from (select * from src) a join (select * from src1) b on (a.key = b.key)) t; diff --git ql/src/test/queries/clientnegative/ambiguous_col2.q ql/src/test/queries/clientnegative/ambiguous_col2.q deleted file mode 100644 index 33d4aed..0000000 --- ql/src/test/queries/clientnegative/ambiguous_col2.q +++ /dev/null @@ -1,3 +0,0 @@ -set hive.support.quoted.identifiers=none; --- DOT -explain select * from (select a.`[kv].*`, b.`[kv].*` from (select * from src) a join (select * from src1) b on (a.key = b.key)) t; diff --git ql/src/test/queries/clientpositive/complex_alias.q ql/src/test/queries/clientpositive/complex_alias.q new file mode 100644 index 0000000..e2810c3 --- /dev/null +++ ql/src/test/queries/clientpositive/complex_alias.q @@ -0,0 +1,46 @@ +CREATE TABLE agg1 (col0 INT, col1 STRING, col2 DOUBLE); + +INSERT INTO TABLE agg1 select key,value,key from src tablesample (1 rows); + +EXPLAIN +SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ); + +SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ); diff --git ql/src/test/results/clientnegative/ambiguous_col.q.out ql/src/test/results/clientnegative/ambiguous_col.q.out index 237c21f..a2915a4 100644 --- ql/src/test/results/clientnegative/ambiguous_col.q.out +++ ql/src/test/results/clientnegative/ambiguous_col.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10007]: Ambiguous column reference key +FAILED: SemanticException [Error 10007]: Ambiguous column reference key in a diff --git ql/src/test/results/clientnegative/ambiguous_col0.q.out ql/src/test/results/clientnegative/ambiguous_col0.q.out deleted file mode 100644 index 237c21f..0000000 --- ql/src/test/results/clientnegative/ambiguous_col0.q.out +++ /dev/null @@ -1 +0,0 @@ -FAILED: SemanticException [Error 10007]: Ambiguous column reference key diff --git ql/src/test/results/clientnegative/ambiguous_col1.q.out ql/src/test/results/clientnegative/ambiguous_col1.q.out deleted file mode 100644 index 237c21f..0000000 --- ql/src/test/results/clientnegative/ambiguous_col1.q.out +++ /dev/null @@ -1 +0,0 @@ -FAILED: SemanticException [Error 10007]: Ambiguous column reference key diff --git ql/src/test/results/clientnegative/ambiguous_col2.q.out ql/src/test/results/clientnegative/ambiguous_col2.q.out deleted file mode 100644 index 237c21f..0000000 --- ql/src/test/results/clientnegative/ambiguous_col2.q.out +++ /dev/null @@ -1 +0,0 @@ -FAILED: SemanticException [Error 10007]: Ambiguous column reference key diff --git ql/src/test/results/clientpositive/ambiguous_col.q.out ql/src/test/results/clientpositive/ambiguous_col.q.out index e8760f1..d583162 100644 --- ql/src/test/results/clientpositive/ambiguous_col.q.out +++ ql/src/test/results/clientpositive/ambiguous_col.q.out @@ -53,8 +53,8 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 + expressions: _col0 (type: string), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -124,8 +124,8 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 + expressions: _col0 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -195,8 +195,8 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 + expressions: _col0 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git ql/src/test/results/clientpositive/complex_alias.q.out ql/src/test/results/clientpositive/complex_alias.q.out new file mode 100644 index 0000000..d8264bd --- /dev/null +++ ql/src/test/results/clientpositive/complex_alias.q.out @@ -0,0 +1,269 @@ +PREHOOK: query: CREATE TABLE agg1 (col0 INT, col1 STRING, col2 DOUBLE) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@agg1 +POSTHOOK: query: CREATE TABLE agg1 (col0 INT, col1 STRING, col2 DOUBLE) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@agg1 +PREHOOK: query: INSERT INTO TABLE agg1 select key,value,key from src tablesample (1 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@agg1 +POSTHOOK: query: INSERT INTO TABLE agg1 select key,value,key from src tablesample (1 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@agg1 +POSTHOOK: Lineage: agg1.col0 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: agg1.col1 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: agg1.col2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +Warning: Shuffle Join JOIN[19][tables = [single_use_subq12, single_use_subq11]] in Stage 'Stage-2:MAPRED' is a cross product +PREHOOK: query: EXPLAIN +SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-4 + Stage-4 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: agg1 + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (col0 = col0) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: '42' (type: string), col0 (type: int) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Union + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col2 (type: int), _col1 (type: string) + outputColumnNames: _col2, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + keys: _col2 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + TableScan + alias: agg1 + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (col0 = col0) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: '41' (type: string), col0 (type: int) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Union + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col2 (type: int), _col1 (type: string) + outputColumnNames: _col2, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + keys: _col2 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col1 (type: string) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string) + TableScan + Reduce Output Operator + sort order: + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {VALUE._col1} {VALUE._col2} + outputColumnNames: _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: agg1 + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col0 (type: int), col2 (type: double) + outputColumnNames: col0, col2 + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(col2) + keys: col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join JOIN[19][tables = [single_use_subq12, single_use_subq11]] in Stage 'Stage-2:MAPRED' is a cross product +PREHOOK: query: SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ) +PREHOOK: type: QUERY +PREHOOK: Input: default@agg1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT single_use_subq11.a1 AS a1, + single_use_subq11.a2 AS a2 +FROM (SELECT Sum(agg1.col2) AS a1 + FROM agg1 + GROUP BY agg1.col0) single_use_subq12 + JOIN (SELECT alias.a2 AS a0, + alias.a1 AS a1, + alias.a1 AS a2 + FROM (SELECT agg1.col1 AS a0, + '42' AS a1, + agg1.col0 AS a2 + FROM agg1 + UNION ALL + SELECT agg1.col1 AS a0, + '41' AS a1, + agg1.col0 AS a2 + FROM agg1) alias + GROUP BY alias.a2, + alias.a1) single_use_subq11 + ON ( single_use_subq11.a0 = single_use_subq11.a0 ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@agg1 +#### A masked pattern was here #### +42 42 +41 41