diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 9bd90a9..ab4e9fa 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7368,17 +7368,21 @@ private Operator genUnionPlan(String unionalias, String leftalias, } RowResolver rowResolver = new RowResolver(); + Map columnExprMap = new HashMap(); + List colName = new ArrayList(); for (int i = 0; i < columns.size(); i++) { String name = getColumnInternalName(i); - rowResolver.put(origInputAlias, name, new ColumnInfo(name, columns.get(i) - .getTypeInfo(), "", false)); + ColumnInfo col = new ColumnInfo(name, columns.get(i) + .getTypeInfo(), "", false); + rowResolver.put(origInputAlias, name, col); colName.add(name); + columnExprMap.put(name, columns.get(i)); } Operator newInputOp = OperatorFactory.getAndMakeChild( new SelectDesc(columns, colName), new RowSchema(rowResolver.getColumnInfos()), - origInputOp); + columnExprMap, origInputOp); return putOpInsertMap(newInputOp, rowResolver); } diff --git ql/src/test/results/clientpositive/union_remove_14.q.out ql/src/test/results/clientpositive/union_remove_14.q.out index 43d3ae8..9c408e9 100644 --- ql/src/test/results/clientpositive/union_remove_14.q.out +++ ql/src/test/results/clientpositive/union_remove_14.q.out @@ -67,18 +67,14 @@ ABSTRACT SYNTAX TREE: STAGE DEPENDENCIES: Stage-8 is a root stage - Stage-6 depends on stages: Stage-1, Stage-8, Stage-9, Stage-10 , consists of Stage-3, Stage-2, Stage-4 + Stage-6 depends on stages: Stage-8, Stage-9 , consists of Stage-3, Stage-2, Stage-4 Stage-3 Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 Stage-2 Stage-4 Stage-5 depends on stages: Stage-4 - Stage-11 is a root stage , consists of Stage-12, Stage-13, Stage-1 - Stage-12 has a backup stage: Stage-1 - Stage-9 depends on stages: Stage-12 - Stage-13 has a backup stage: Stage-1 - Stage-10 depends on stages: Stage-13 - Stage-1 + Stage-10 is a root stage + Stage-9 depends on stages: Stage-10 STAGE PLANS: Stage: Stage-8 @@ -91,7 +87,7 @@ STAGE PLANS: expressions: expr: key type: string - expr: UDFToLong(1) + expr: UDFToLong(UDFToString(1)) type: bigint outputColumnNames: _col0, _col1 File Output Operator @@ -134,66 +130,7 @@ STAGE PLANS: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-11 - Conditional Operator - - Stage: Stage-12 - Map Reduce Local Work - Alias -> Map Local Tables: - null-subquery2:c-subquery2:b - Fetch Operator - limit: -1 - Alias -> Map Local Operator Tree: - null-subquery2:c-subquery2:b - TableScan - alias: b - HashTable Sink Operator - condition expressions: - 0 {key} - 1 {val} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - Position of Big Table: 0 - - Stage: Stage-9 - Map Reduce - Alias -> Map Operator Tree: - null-subquery2:c-subquery2:a - TableScan - alias: a - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {key} - 1 {val} - handleSkewJoin: false - keys: - 0 [Column[key]] - 1 [Column[key]] - outputColumnNames: _col0, _col5 - Position of Big Table: 0 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col5) - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 - Local Work: - Map Reduce Local Work - - Stage: Stage-13 + Stage: Stage-10 Map Reduce Local Work Alias -> Map Local Tables: null-subquery2:c-subquery2:a @@ -213,7 +150,7 @@ STAGE PLANS: 1 [Column[key]] Position of Big Table: 1 - Stage: Stage-10 + Stage: Stage-9 Map Reduce Alias -> Map Operator Tree: null-subquery2:c-subquery2:b @@ -249,64 +186,6 @@ STAGE PLANS: Local Work: Map Reduce Local Work - Stage: Stage-1 - Map Reduce - Alias -> Map Operator Tree: - null-subquery2:c-subquery2:a - TableScan - alias: a - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 0 - value expressions: - expr: key - type: string - null-subquery2:c-subquery2:b - TableScan - alias: b - Reduce Output Operator - key expressions: - expr: key - type: string - sort order: + - Map-reduce partition columns: - expr: key - type: string - tag: 1 - value expressions: - expr: val - type: string - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {VALUE._col0} - 1 {VALUE._col1} - handleSkewJoin: false - outputColumnNames: _col0, _col5 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col5) - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM diff --git ql/src/test/results/clientpositive/union_remove_2.q.out ql/src/test/results/clientpositive/union_remove_2.q.out index 5aa55a5..ec044cb 100644 --- ql/src/test/results/clientpositive/union_remove_2.q.out +++ ql/src/test/results/clientpositive/union_remove_2.q.out @@ -78,8 +78,8 @@ STAGE PLANS: expressions: expr: key type: string - expr: 2 - type: int + expr: UDFToLong(2) + type: bigint outputColumnNames: _col0, _col1 File Output Operator compressed: false @@ -168,8 +168,8 @@ STAGE PLANS: expressions: expr: key type: string - expr: 1 - type: int + expr: UDFToLong(1) + type: bigint outputColumnNames: _col0, _col1 File Output Operator compressed: false diff --git ql/src/test/results/clientpositive/union_remove_24.q.out ql/src/test/results/clientpositive/union_remove_24.q.out index 7f1741a..40199e3 100644 --- ql/src/test/results/clientpositive/union_remove_24.q.out +++ ql/src/test/results/clientpositive/union_remove_24.q.out @@ -103,33 +103,19 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Select Operator expressions: - expr: UDFToLong(_col0) - type: bigint + expr: UDFToDouble(UDFToLong(_col0)) + type: double expr: _col1 type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: UDFToDouble(_col0) - type: double - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: double - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 Stage: Stage-0 Move Operator @@ -190,21 +176,14 @@ STAGE PLANS: expr: _col1 type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: double - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 diff --git ql/src/test/results/clientpositive/union_remove_5.q.out ql/src/test/results/clientpositive/union_remove_5.q.out index b628ad1..79ed6a7 100644 --- ql/src/test/results/clientpositive/union_remove_5.q.out +++ ql/src/test/results/clientpositive/union_remove_5.q.out @@ -85,31 +85,17 @@ STAGE PLANS: expressions: expr: key type: string - expr: 2 - type: int + expr: UDFToLong(2) + type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 Stage: Stage-6 Conditional Operator @@ -211,11 +197,26 @@ STAGE PLANS: expr: _col1 type: bigint outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 + + Stage: Stage-8 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1-subquery2:a-subquery1-subquery2:inputtbl1 + TableScan + alias: inputtbl1 Select Operator expressions: - expr: _col0 + expr: key type: string - expr: _col1 + expr: UDFToLong(1) type: bigint outputColumnNames: _col0, _col1 File Output Operator @@ -227,42 +228,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-8 - Map Reduce - Alias -> Map Operator Tree: - null-subquery1-subquery2:a-subquery1-subquery2:inputtbl1 - TableScan - alias: inputtbl1 - Select Operator - expressions: - expr: key - type: string - expr: 1 - type: int - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * diff --git ql/src/test/results/clientpositive/union_remove_8.q.out ql/src/test/results/clientpositive/union_remove_8.q.out index c7bc105..b999476 100644 --- ql/src/test/results/clientpositive/union_remove_8.q.out +++ ql/src/test/results/clientpositive/union_remove_8.q.out @@ -82,31 +82,17 @@ STAGE PLANS: expressions: expr: key type: string - expr: 2 - type: int + expr: UDFToLong(2) + type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 Stage: Stage-0 Move Operator @@ -167,11 +153,26 @@ STAGE PLANS: expr: _col1 type: bigint outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1-subquery2:a-subquery1-subquery2:inputtbl1 + TableScan + alias: inputtbl1 Select Operator expressions: - expr: _col0 + expr: key type: string - expr: _col1 + expr: UDFToLong(1) type: bigint outputColumnNames: _col0, _col1 File Output Operator @@ -183,42 +184,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-3 - Map Reduce - Alias -> Map Operator Tree: - null-subquery1-subquery2:a-subquery1-subquery2:inputtbl1 - TableScan - alias: inputtbl1 - Select Operator - expressions: - expr: key - type: string - expr: 1 - type: int - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * diff --git ql/src/test/results/clientpositive/union_remove_9.q.out ql/src/test/results/clientpositive/union_remove_9.q.out index 0c64654..90d669b 100644 --- ql/src/test/results/clientpositive/union_remove_9.q.out +++ ql/src/test/results/clientpositive/union_remove_9.q.out @@ -96,31 +96,17 @@ STAGE PLANS: expressions: expr: _col0 type: string - expr: _col1 - type: int + expr: UDFToLong(_col1) + type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 null-subquery2:b-subquery2-subquery2:a-subquery2:inputtbl1 TableScan alias: inputtbl1 @@ -136,31 +122,17 @@ STAGE PLANS: expressions: expr: _col0 type: string - expr: _col1 - type: int + expr: UDFToLong(_col1) + type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 Stage: Stage-6 Conditional Operator @@ -242,21 +214,14 @@ STAGE PLANS: expr: _col1 type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 PREHOOK: query: insert overwrite table outputTbl1