diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java index 69fff0e..fcff842 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java @@ -20,10 +20,12 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import org.apache.hadoop.hive.ql.plan.CollectDesc; import org.apache.hadoop.hive.ql.plan.DummyStoreDesc; import org.apache.hadoop.hive.ql.plan.ExtractDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; import org.apache.hadoop.hive.ql.plan.ForwardDesc; @@ -251,12 +253,32 @@ public static void makeChild( * Returns an operator given the conf and a list of parent operators. */ public static Operator getAndMakeChild(T conf, + RowSchema rwsch, Map colExprMap, Operator... oplist) { + Operator ret = getAndMakeChild(conf, rwsch, oplist); + ret.setColumnExprMap(colExprMap); + return (ret); + } + + /** + * Returns an operator given the conf and a list of parent operators. + */ + public static Operator getAndMakeChild(T conf, RowSchema rwsch, List> oplist) { Operator ret = getAndMakeChild(conf, oplist); ret.setSchema(rwsch); return (ret); } + /** + * Returns an operator given the conf and a list of parent operators. + */ + public static Operator getAndMakeChild(T conf, + RowSchema rwsch, Map colExprMap, List> oplist) { + Operator ret = getAndMakeChild(conf, rwsch, oplist); + ret.setColumnExprMap(colExprMap); + return (ret); + } + private OperatorFactory() { // prevent instantiation } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java index 49cefcc..cb3d38c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java @@ -188,7 +188,8 @@ private void pushOperatorsAboveUnion(UnionOperator union, OperatorDesc cloneDesc = (OperatorDesc)originalOp.getConf().clone(); Operator cloneOp = - OperatorFactory.getAndMakeChild(cloneDesc, originalOp.getSchema(), parents.get(p)); + OperatorFactory.getAndMakeChild(cloneDesc, originalOp.getSchema(), + originalOp.getColumnExprMap(), parents.get(p)); parents.set(p, cloneOp); } } diff --git ql/src/test/results/clientpositive/union_remove_22.q.out ql/src/test/results/clientpositive/union_remove_22.q.out index ea76dfb..2c77827 100644 --- ql/src/test/results/clientpositive/union_remove_22.q.out +++ ql/src/test/results/clientpositive/union_remove_22.q.out @@ -109,24 +109,17 @@ STAGE PLANS: type: string expr: _col1 type: bigint - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + expr: _col1 + type: bigint + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 Stage: Stage-0 Move Operator @@ -186,24 +179,17 @@ STAGE PLANS: type: string expr: _col1 type: bigint - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + expr: _col1 + type: bigint + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 PREHOOK: query: insert overwrite table outputTbl1 @@ -359,35 +345,19 @@ STAGE PLANS: expressions: expr: _col0 type: string - expr: _col1 + expr: UDFToLong(concat(_col1, _col1)) type: bigint - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: concat(_col1, _col1) - type: string - expr: concat(_col1, _col1) - type: string - outputColumnNames: _col0, _col1, _col2 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - expr: UDFToLong(_col2) - type: bigint - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + expr: UDFToLong(concat(_col1, _col1)) + type: bigint + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 Stage: Stage-0 Move Operator @@ -445,35 +415,19 @@ STAGE PLANS: expressions: expr: _col0 type: string - expr: _col1 + expr: UDFToLong(concat(_col1, _col1)) type: bigint - outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: concat(_col1, _col1) - type: string - expr: concat(_col1, _col1) - type: string - outputColumnNames: _col0, _col1, _col2 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - expr: UDFToLong(_col2) - type: bigint - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + expr: UDFToLong(concat(_col1, _col1)) + type: bigint + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 PREHOOK: query: insert overwrite table outputTbl1 diff --git ql/src/test/results/clientpositive/union_remove_23.q.out ql/src/test/results/clientpositive/union_remove_23.q.out index a162799..f53d5b2 100644 --- ql/src/test/results/clientpositive/union_remove_23.q.out +++ ql/src/test/results/clientpositive/union_remove_23.q.out @@ -115,21 +115,14 @@ STAGE PLANS: expr: _col1 type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 Stage: Stage-0 Move Operator @@ -185,26 +178,21 @@ STAGE PLANS: expr: _col0 type: string outputColumnNames: _col0 - Select Operator - expressions: + Group By Operator + aggregations: + expr: count(1) + bucketGroup: false + keys: expr: _col0 type: string - outputColumnNames: _col0 - Group By Operator - aggregations: - expr: count(1) - bucketGroup: false - keys: - expr: _col0 - type: string - mode: hash - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 0 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + mode: hash + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat Stage: Stage-3 Map Reduce @@ -239,21 +227,14 @@ STAGE PLANS: expr: _col1 type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 PREHOOK: query: insert overwrite table outputTbl1 diff --git ql/src/test/results/clientpositive/union_remove_3.q.out ql/src/test/results/clientpositive/union_remove_3.q.out index 5bbee76..0860e75 100644 --- ql/src/test/results/clientpositive/union_remove_3.q.out +++ ql/src/test/results/clientpositive/union_remove_3.q.out @@ -84,24 +84,17 @@ STAGE PLANS: expressions: expr: _col0 type: string - expr: _col1 - type: int + expr: UDFToLong(_col1) + type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 null-subquery1-subquery2:a-subquery1-subquery2:inputtbl1 TableScan alias: inputtbl1 @@ -117,24 +110,17 @@ STAGE PLANS: expressions: expr: _col0 type: string - expr: _col1 - type: int + expr: UDFToLong(_col1) + type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 null-subquery2:a-subquery2:inputtbl1 TableScan alias: inputtbl1 @@ -150,24 +136,17 @@ STAGE PLANS: expressions: expr: _col0 type: string - expr: _col1 - type: int + expr: UDFToLong(_col1) + type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: UDFToLong(_col1) - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 Stage: Stage-0 Move Operator @@ -179,7 +158,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git ql/src/test/results/clientpositive/union_remove_4.q.out ql/src/test/results/clientpositive/union_remove_4.q.out index e676c20..3cffe90 100644 --- ql/src/test/results/clientpositive/union_remove_4.q.out +++ ql/src/test/results/clientpositive/union_remove_4.q.out @@ -115,21 +115,14 @@ STAGE PLANS: expr: _col1 type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 Stage: Stage-6 Conditional Operator @@ -231,21 +224,14 @@ STAGE PLANS: expr: _col1 type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl1 PREHOOK: query: insert overwrite table outputTbl1 diff --git ql/src/test/results/clientpositive/union_remove_7.q.out ql/src/test/results/clientpositive/union_remove_7.q.out index 777c141..81699f9 100644 --- ql/src/test/results/clientpositive/union_remove_7.q.out +++ ql/src/test/results/clientpositive/union_remove_7.q.out @@ -114,21 +114,14 @@ STAGE PLANS: expr: _col1 type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 Stage: Stage-0 Move Operator @@ -189,21 +182,14 @@ STAGE PLANS: expr: _col1 type: bigint outputColumnNames: _col0, _col1 - Select Operator - expressions: - expr: _col0 - type: string - expr: _col1 - type: bigint - outputColumnNames: _col0, _col1 - File Output Operator - compressed: false - GlobalTableId: 1 - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1 PREHOOK: query: insert overwrite table outputTbl1