diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 7c09fcc..f52350d 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -1024,7 +1024,7 @@ private int executeClientInternal(String commands) { } command = ""; } - if (SessionState.get() != null) { + if (rc == 0 && SessionState.get() != null) { SessionState.get().setLastCommand(null); // reset } return rc; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 3a613a2..fd4ade1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -859,49 +859,30 @@ private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode ASTNode tabref = getQB().getAliases().isEmpty() ? null : getQB().getParseInfo() .getSrcForAlias(getQB().getAliases().get(0)); - for (Map.Entry lEntry : leftmap.entrySet()) { - String field = lEntry.getKey(); + + // 3. construct Union Output RR using original left & right Input + RowResolver unionoutRR = new RowResolver(); + + Iterator> lIter = leftmap.entrySet().iterator(); + Iterator> rIter = rightmap.entrySet().iterator(); + while (lIter.hasNext()) { + Map.Entry lEntry = lIter.next(); + Map.Entry rEntry = rIter.next(); ColumnInfo lInfo = lEntry.getValue(); - ColumnInfo rInfo = rightmap.get(field); - if (rInfo == null) { - throw new SemanticException(SemanticAnalyzer.generateErrorMessage(tabref, - "Schema of both sides of union should match. " + rightalias - + " does not have the field " + field)); - } - if (lInfo == null) { - throw new SemanticException(SemanticAnalyzer.generateErrorMessage(tabref, - "Schema of both sides of union should match. " + leftalias - + " does not have the field " + field)); - } - if (!lInfo.getInternalName().equals(rInfo.getInternalName())) { - throw new CalciteSemanticException(SemanticAnalyzer.generateErrorMessage( - tabref, - "Schema of both sides of union should match: field " + field + ":" - + " appears on the left side of the UNION at column position: " - + SemanticAnalyzer.getPositionFromInternalName(lInfo.getInternalName()) - + ", and on the right side of the UNION at column position: " - + SemanticAnalyzer.getPositionFromInternalName(rInfo.getInternalName()) - + ". Column positions should match for a UNION")); - } - // try widening coversion, otherwise fail union + ColumnInfo rInfo = rEntry.getValue(); + + String field = lEntry.getKey(); + // try widening conversion, otherwise fail union TypeInfo commonTypeInfo = FunctionRegistry.getCommonClassForUnionAll(lInfo.getType(), rInfo.getType()); if (commonTypeInfo == null) { - throw new CalciteSemanticException(SemanticAnalyzer.generateErrorMessage(tabref, - "Schema of both sides of union should match: Column " + field + " is of type " - + lInfo.getType().getTypeName() + " on first table and type " - + rInfo.getType().getTypeName() + " on second table")); + throw new SemanticException(generateErrorMessage(tabref, + "Schema of both sides of union should match: Column " + field + + " is of type " + lInfo.getType().getTypeName() + + " on first table and type " + rInfo.getType().getTypeName() + + " on second table")); } - } - - // 3. construct Union Output RR using original left & right Input - RowResolver unionoutRR = new RowResolver(); - for (Map.Entry lEntry : leftmap.entrySet()) { - String field = lEntry.getKey(); - ColumnInfo lInfo = lEntry.getValue(); - ColumnInfo rInfo = rightmap.get(field); ColumnInfo unionColInfo = new ColumnInfo(lInfo); - unionColInfo.setTabAlias(unionalias); unionColInfo.setType(FunctionRegistry.getCommonClassForUnionAll(lInfo.getType(), rInfo.getType())); unionoutRR.put(unionalias, field, unionColInfo); @@ -2637,6 +2618,7 @@ public RexNode apply(RelDataTypeField input) { LOG.debug("Created Plan for Query Block " + qb.getId()); } + setQB(qb); return srcRel; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 2466d78..8d9ce98 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -557,7 +557,7 @@ private void doPhase1GetAllAggregations(ASTNode expressionTree, public static String generateErrorMessage(ASTNode ast, String message) { StringBuilder sb = new StringBuilder(); if (ast == null) { - sb.append("The abstract syntax tree is null"); + sb.append(message).append(". Cannot tell the position of null AST."); return sb.toString(); } sb.append(ast.getLine()); @@ -8755,30 +8755,19 @@ private Operator genUnionPlan(String unionalias, String leftalias, if (leftmap.size() != rightmap.size()) { throw new SemanticException("Schema of both sides of union should match."); } - for (Map.Entry lEntry : leftmap.entrySet()) { - String field = lEntry.getKey(); + + RowResolver unionoutRR = new RowResolver(); + + Iterator> lIter = leftmap.entrySet().iterator(); + Iterator> rIter = rightmap.entrySet().iterator(); + while (lIter.hasNext()) { + Map.Entry lEntry = lIter.next(); + Map.Entry rEntry = rIter.next(); ColumnInfo lInfo = lEntry.getValue(); - ColumnInfo rInfo = rightmap.get(field); - if (rInfo == null) { - throw new SemanticException(generateErrorMessage(tabref, - "Schema of both sides of union should match. " + rightalias - + " does not have the field " + field)); - } - if (lInfo == null) { - throw new SemanticException(generateErrorMessage(tabref, - "Schema of both sides of union should match. " + leftalias - + " does not have the field " + field)); - } - if (!lInfo.getInternalName().equals(rInfo.getInternalName())) { - throw new SemanticException(generateErrorMessage(tabref, - "Schema of both sides of union should match: field " + field + ":" - + " appears on the left side of the UNION at column position: " + - getPositionFromInternalName(lInfo.getInternalName()) - + ", and on the right side of the UNION at column position: " + - getPositionFromInternalName(rInfo.getInternalName()) - + ". Column positions should match for a UNION")); - } - // try widening coversion, otherwise fail union + ColumnInfo rInfo = rEntry.getValue(); + + String field = lEntry.getKey(); // use left alias (~mysql, postgresql) + // try widening conversion, otherwise fail union TypeInfo commonTypeInfo = FunctionRegistry.getCommonClassForUnionAll(lInfo.getType(), rInfo.getType()); if (commonTypeInfo == null) { @@ -8788,14 +8777,6 @@ private Operator genUnionPlan(String unionalias, String leftalias, + " on first table and type " + rInfo.getType().getTypeName() + " on second table")); } - } - - // construct the forward operator - RowResolver unionoutRR = new RowResolver(); - for (Map.Entry lEntry : leftmap.entrySet()) { - String field = lEntry.getKey(); - ColumnInfo lInfo = lEntry.getValue(); - ColumnInfo rInfo = rightmap.get(field); ColumnInfo unionColInfo = new ColumnInfo(lInfo); unionColInfo.setType(FunctionRegistry.getCommonClassForUnionAll(lInfo.getType(), rInfo.getType())); @@ -8921,17 +8902,22 @@ private Operator genUnionPlan(String unionalias, String leftalias, String origInputAlias, RowResolver unionoutRR, String unionalias) throws SemanticException { + HashMap fieldMap = unionoutRR.getFieldMap(unionalias); + + Iterator oIter = origInputFieldMap.values().iterator(); + Iterator uIter = fieldMap.values().iterator(); + List columns = new ArrayList(); boolean needsCast = false; - for (Map.Entry unionEntry : unionoutRR.getFieldMap(unionalias).entrySet()) { - String field = unionEntry.getKey(); - ColumnInfo lInfo = origInputFieldMap.get(field); - ExprNodeDesc column = new ExprNodeColumnDesc(lInfo.getType(), lInfo.getInternalName(), - lInfo.getTabAlias(), lInfo.getIsVirtualCol(), lInfo.isSkewedCol()); - if (!lInfo.getType().equals(unionEntry.getValue().getType())) { + while (oIter.hasNext()) { + ColumnInfo oInfo = oIter.next(); + ColumnInfo uInfo = uIter.next(); + ExprNodeDesc column = new ExprNodeColumnDesc(oInfo.getType(), oInfo.getInternalName(), + oInfo.getTabAlias(), oInfo.getIsVirtualCol(), oInfo.isSkewedCol()); + if (!oInfo.getType().equals(uInfo.getType())) { needsCast = true; column = ParseUtils.createConversionCast( - column, (PrimitiveTypeInfo)unionEntry.getValue().getType()); + column, (PrimitiveTypeInfo)uInfo.getType()); } columns.add(column); } diff --git a/ql/src/test/queries/clientnegative/union3.q b/ql/src/test/queries/clientnegative/union3.q deleted file mode 100644 index ce65747..0000000 --- a/ql/src/test/queries/clientnegative/union3.q +++ /dev/null @@ -1,5 +0,0 @@ --- Ensure that UNION ALL columns are in the correct order on both sides --- Ensure that the appropriate error message is propagated -CREATE TABLE IF NOT EXISTS union3 (bar int, baz int); -SELECT * FROM ( SELECT f.bar, f.baz FROM union3 f UNION ALL SELECT b.baz, b.bar FROM union3 b ) c; -DROP TABLE union3; diff --git a/ql/src/test/queries/clientpositive/union35.q b/ql/src/test/queries/clientpositive/union35.q new file mode 100644 index 0000000..9445711 --- /dev/null +++ b/ql/src/test/queries/clientpositive/union35.q @@ -0,0 +1,15 @@ +set hive.cbo.enable=true; + +select * from ( + select * from ( select 1 as id , 'foo' as str_1 from src tablesample(5 rows)) f + union all + select * from ( select 2 as id , 'bar' as str_2 from src tablesample(5 rows)) g +) e ; + +set hive.cbo.enable=false; + +select * from ( + select * from ( select 1 as id , 'foo' as str_1 from src tablesample(5 rows)) f + union all + select * from ( select 2 as id , 'bar' as str_2 from src tablesample(5 rows)) g +) e ; diff --git a/ql/src/test/results/clientnegative/union3.q.out b/ql/src/test/results/clientnegative/union3.q.out deleted file mode 100644 index de1c62b..0000000 --- a/ql/src/test/results/clientnegative/union3.q.out +++ /dev/null @@ -1,13 +0,0 @@ -PREHOOK: query: -- Ensure that UNION ALL columns are in the correct order on both sides --- Ensure that the appropriate error message is propagated -CREATE TABLE IF NOT EXISTS union3 (bar int, baz int) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@union3 -POSTHOOK: query: -- Ensure that UNION ALL columns are in the correct order on both sides --- Ensure that the appropriate error message is propagated -CREATE TABLE IF NOT EXISTS union3 (bar int, baz int) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@union3 -FAILED: SemanticException 2:85 Schema of both sides of union should match: field bar: appears on the left side of the UNION at column position: 0, and on the right side of the UNION at column position: 1. Column positions should match for a UNION. Error encountered near token 'union3' diff --git a/ql/src/test/results/clientpositive/union35.q.out b/ql/src/test/results/clientpositive/union35.q.out new file mode 100644 index 0000000..2bf0327 --- /dev/null +++ b/ql/src/test/results/clientpositive/union35.q.out @@ -0,0 +1,52 @@ +PREHOOK: query: select * from ( + select * from ( select 1 as id , 'foo' as str_1 from src tablesample(5 rows)) f + union all + select * from ( select 2 as id , 'bar' as str_2 from src tablesample(5 rows)) g +) e +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from ( + select * from ( select 1 as id , 'foo' as str_1 from src tablesample(5 rows)) f + union all + select * from ( select 2 as id , 'bar' as str_2 from src tablesample(5 rows)) g +) e +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 foo +2 bar +1 foo +2 bar +1 foo +2 bar +1 foo +2 bar +1 foo +2 bar +PREHOOK: query: select * from ( + select * from ( select 1 as id , 'foo' as str_1 from src tablesample(5 rows)) f + union all + select * from ( select 2 as id , 'bar' as str_2 from src tablesample(5 rows)) g +) e +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from ( + select * from ( select 1 as id , 'foo' as str_1 from src tablesample(5 rows)) f + union all + select * from ( select 2 as id , 'bar' as str_2 from src tablesample(5 rows)) g +) e +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 foo +2 bar +1 foo +2 bar +1 foo +2 bar +1 foo +2 bar +1 foo +2 bar