Index: ql/src/test/results/clientpositive/union24.q.out =================================================================== --- ql/src/test/results/clientpositive/union24.q.out (revision 0) +++ ql/src/test/results/clientpositive/union24.q.out (revision 0) @@ -0,0 +1,960 @@ +PREHOOK: query: explain extended + select * from ( + select 1234 from src limit 10 + union all + select '1234' from src limit 10 + ) t +PREHOOK: type: QUERY +POSTHOOK: query: explain extended + select * from ( + select 1234 from src limit 10 + union all + select '1234' from src limit 10 + ) t +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR 1234)) (TOK_LIMIT 10))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR '1234')) (TOK_LIMIT 10)))) t)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3 + Stage-3 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + null-subquery2:t-subquery2:src + TableScan + alias: src + GatherStats: false + Select Operator + expressions: + expr: '1234' + type: string + outputColumnNames: _col0 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + Needs Tagging: false + Path -> Alias: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src [null-subquery2:t-subquery2:src] + Path -> Partition: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-mr-10002 + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-mr-10002 + Union + Select Operator + expressions: + expr: _col0 + type: string + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-ext-10001 + NumFilesPerFileSink: 1 + Stats Publishing Key Prefix: file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-ext-10001/ + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + serialization.format 1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-mr-10003 + Union + Select Operator + expressions: + expr: _col0 + type: string + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-ext-10001 + NumFilesPerFileSink: 1 + Stats Publishing Key Prefix: file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-ext-10001/ + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + serialization.format 1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: + file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-mr-10002 [file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-mr-10002] + file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-mr-10003 [file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-mr-10003] + Path -> Partition: + file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-mr-10002 + Partition + base file name: -mr-10002 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-mr-10003 + Partition + base file name: -mr-10003 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types int + escape.delim \ + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types int + escape.delim \ + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1:t-subquery1:src + TableScan + alias: src + GatherStats: false + Select Operator + expressions: + expr: 1234 + type: int + outputColumnNames: _col0 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: int + Needs Tagging: false + Path -> Alias: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src [null-subquery1:t-subquery1:src] + Path -> Partition: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-36_258_6839950498648203652/-mr-10003 + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types int + escape.delim \ + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: select * from ( + select 1234 from src limit 10 + union all + select '1234' from src limit 10 +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: file:/tmp/franklin/hive_2011-07-26_14-31-36_556_3919078646274882711/-mr-10000 +POSTHOOK: query: select * from ( + select 1234 from src limit 10 + union all + select '1234' from src limit 10 +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: file:/tmp/franklin/hive_2011-07-26_14-31-36_556_3919078646274882711/-mr-10000 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +1234 +PREHOOK: query: explain extended + select * from ( + select 1234, 'some-string', cast(2345 as bigint) from src limit 10 + union all + select 'other-string', 1, 0 from src limit 10 + ) t +PREHOOK: type: QUERY +POSTHOOK: query: explain extended + select * from ( + select 1234, 'some-string', cast(2345 as bigint) from src limit 10 + union all + select 'other-string', 1, 0 from src limit 10 + ) t +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR 1234) (TOK_SELEXPR 'some-string') (TOK_SELEXPR (TOK_FUNCTION TOK_BIGINT 2345))) (TOK_LIMIT 10))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR 'other-string') (TOK_SELEXPR 1) (TOK_SELEXPR 0)) (TOK_LIMIT 10)))) t)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3 + Stage-3 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + null-subquery2:t-subquery2:src + TableScan + alias: src + GatherStats: false + Select Operator + expressions: + expr: 'other-string' + type: string + expr: 1 + type: int + expr: 0 + type: int + outputColumnNames: _col0, _col1, _col2 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: int + expr: _col2 + type: int + Needs Tagging: false + Path -> Alias: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src [null-subquery2:t-subquery2:src] + Path -> Partition: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-mr-10002 + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,int,int + escape.delim \ + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-mr-10002 + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col2 + type: bigint + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-ext-10001 + NumFilesPerFileSink: 1 + Stats Publishing Key Prefix: file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-ext-10001/ + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string:string:bigint + serialization.format 1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-mr-10003 + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col2 + type: bigint + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-ext-10001 + NumFilesPerFileSink: 1 + Stats Publishing Key Prefix: file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-ext-10001/ + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string:string:bigint + serialization.format 1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: + file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-mr-10002 [file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-mr-10002] + file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-mr-10003 [file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-mr-10003] + Path -> Partition: + file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-mr-10002 + Partition + base file name: -mr-10002 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,int,int + escape.delim \ + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,int,int + escape.delim \ + file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-mr-10003 + Partition + base file name: -mr-10003 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types int,string,bigint + escape.delim \ + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types int,string,bigint + escape.delim \ + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1:t-subquery1:src + TableScan + alias: src + GatherStats: false + Select Operator + expressions: + expr: 1234 + type: int + expr: 'some-string' + type: string + expr: UDFToLong(2345) + type: bigint + outputColumnNames: _col0, _col1, _col2 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: int + expr: _col1 + type: string + expr: _col2 + type: bigint + Needs Tagging: false + Path -> Alias: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src [null-subquery1:t-subquery1:src] + Path -> Partition: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-45_860_2524383031437063319/-mr-10003 + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types int,string,bigint + escape.delim \ + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: select * from ( + select 1234, 'some-string', cast(2345 as bigint) from src limit 10 + union all + select 'other-string', 1, 0 from src limit 10 +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: file:/tmp/franklin/hive_2011-07-26_14-31-46_003_8605801476729166465/-mr-10000 +POSTHOOK: query: select * from ( + select 1234, 'some-string', cast(2345 as bigint) from src limit 10 + union all + select 'other-string', 1, 0 from src limit 10 +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: file:/tmp/franklin/hive_2011-07-26_14-31-46_003_8605801476729166465/-mr-10000 +other-string 1 0 +other-string 1 0 +other-string 1 0 +other-string 1 0 +other-string 1 0 +other-string 1 0 +other-string 1 0 +other-string 1 0 +other-string 1 0 +other-string 1 0 +1234 some-string 2345 +1234 some-string 2345 +1234 some-string 2345 +1234 some-string 2345 +1234 some-string 2345 +1234 some-string 2345 +1234 some-string 2345 +1234 some-string 2345 +1234 some-string 2345 +1234 some-string 2345 +PREHOOK: query: explain extended + select * from ( + select transform(*) using 'tr \t _' as (c) from src limit 10 + union all + select 12345 as c from src limit 10 + ) t +PREHOOK: type: QUERY +POSTHOOK: query: explain extended + select * from ( + select transform(*) using 'tr \t _' as (c) from src limit 10 + union all + select 12345 as c from src limit 10 + ) t +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST TOK_ALLCOLREF) TOK_SERDE TOK_RECORDWRITER 'tr \t _' TOK_SERDE TOK_RECORDREADER (TOK_ALIASLIST c)))) (TOK_LIMIT 10))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR 12345 c)) (TOK_LIMIT 10)))) t)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3 + Stage-3 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + null-subquery2:t-subquery2:src + TableScan + alias: src + GatherStats: false + Select Operator + expressions: + expr: 12345 + type: int + outputColumnNames: _col0 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: int + Needs Tagging: false + Path -> Alias: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src [null-subquery2:t-subquery2:src] + Path -> Partition: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-mr-10002 + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types int + escape.delim \ + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-mr-10002 + Union + Select Operator + expressions: + expr: _col0 + type: string + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-ext-10001 + NumFilesPerFileSink: 1 + Stats Publishing Key Prefix: file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-ext-10001/ + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + serialization.format 1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-mr-10003 + Union + Select Operator + expressions: + expr: _col0 + type: string + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-ext-10001 + NumFilesPerFileSink: 1 + Stats Publishing Key Prefix: file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-ext-10001/ + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + serialization.format 1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: + file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-mr-10002 [file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-mr-10002] + file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-mr-10003 [file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-mr-10003] + Path -> Partition: + file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-mr-10002 + Partition + base file name: -mr-10002 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types int + escape.delim \ + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types int + escape.delim \ + file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-mr-10003 + Partition + base file name: -mr-10003 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + + Stage: Stage-3 + Map Reduce + Alias -> Map Operator Tree: + null-subquery1:t-subquery1:src + TableScan + alias: src + GatherStats: false + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + Transform Operator + command: tr \t _ + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + field.delim 9 + serialization.format 9 + Limit + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: string + Needs Tagging: false + Path -> Alias: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src [null-subquery1:t-subquery1:src] + Path -> Partition: + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src + name default.src + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + transient_lastDdlTime 1311715893 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/franklin/hive_2011-07-26_14-31-55_212_8556106606525901848/-mr-10003 + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: select * from ( + select transform(*) using 'tr \t _' as (c) from src limit 10 + union all + select 12345 as c from src limit 10 +) t +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: file:/tmp/franklin/hive_2011-07-26_14-31-55_345_3158293243050557517/-mr-10000 +POSTHOOK: query: select * from ( + select transform(*) using 'tr \t _' as (c) from src limit 10 + union all + select 12345 as c from src limit 10 +) t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: file:/tmp/franklin/hive_2011-07-26_14-31-55_345_3158293243050557517/-mr-10000 +12345 +12345 +12345 +12345 +12345 +12345 +12345 +12345 +12345 +12345 +238_val_238 +86_val_86 +311_val_311 +27_val_27 +165_val_165 +409_val_409 +255_val_255 +278_val_278 +98_val_98 +484_val_484 Index: ql/src/test/results/compiler/plan/union.q.xml =================================================================== --- ql/src/test/results/compiler/plan/union.q.xml (revision 1151937) +++ ql/src/test/results/compiler/plan/union.q.xml (working copy) @@ -1,5 +1,5 @@ - + @@ -41,7 +41,7 @@ - file:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_04-06-57_227_5273764780066863405/-ext-10001 + file:/data/users/franklin/hive-union/build/ql/scratchdir/hive_2011-07-28_13-48-37_450_4340421797588168189/-ext-10001 @@ -50,7 +50,7 @@ - file:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_04-06-57_227_5273764780066863405/-ext-10000 + file:/data/users/franklin/hive-union/build/ql/scratchdir/hive_2011-07-28_13-48-37_450_4340421797588168189/-ext-10000 1 @@ -92,21 +92,21 @@ - CNTR_NAME_FS_665_NUM_INPUT_ROWS + CNTR_NAME_FS_12_NUM_INPUT_ROWS - CNTR_NAME_FS_665_NUM_OUTPUT_ROWS + CNTR_NAME_FS_12_NUM_OUTPUT_ROWS - CNTR_NAME_FS_665_TIME_TAKEN + CNTR_NAME_FS_12_TIME_TAKEN - CNTR_NAME_FS_665_FATAL_ERROR + CNTR_NAME_FS_12_FATAL_ERROR - FS_665 + FS_12 @@ -160,21 +160,21 @@ - CNTR_NAME_TS_664_NUM_INPUT_ROWS + CNTR_NAME_TS_11_NUM_INPUT_ROWS - CNTR_NAME_TS_664_NUM_OUTPUT_ROWS + CNTR_NAME_TS_11_NUM_OUTPUT_ROWS - CNTR_NAME_TS_664_TIME_TAKEN + CNTR_NAME_TS_11_TIME_TAKEN - CNTR_NAME_TS_664_FATAL_ERROR + CNTR_NAME_TS_11_FATAL_ERROR - TS_664 + TS_11 @@ -195,10 +195,10 @@ - file:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_04-06-57_227_5273764780066863405/-ext-10001 + file:/data/users/franklin/hive-union/build/ql/scratchdir/hive_2011-07-28_13-48-37_450_4340421797588168189/-ext-10001 - file:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_04-06-57_227_5273764780066863405/-ext-10001 + file:/data/users/franklin/hive-union/build/ql/scratchdir/hive_2011-07-28_13-48-37_450_4340421797588168189/-ext-10001 @@ -207,7 +207,7 @@ - file:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_04-06-57_227_5273764780066863405/-ext-10001 + file:/data/users/franklin/hive-union/build/ql/scratchdir/hive_2011-07-28_13-48-37_450_4340421797588168189/-ext-10001 -ext-10001 @@ -280,7 +280,7 @@ true - file:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_04-06-57_227_5273764780066863405/-ext-10000 + file:/data/users/franklin/hive-union/build/ql/scratchdir/hive_2011-07-28_13-48-37_450_4340421797588168189/-ext-10000 ../build/ql/test/data/warehouse/union.out @@ -304,10 +304,10 @@ true - file:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_04-06-57_227_5273764780066863405/-ext-10001 + file:/data/users/franklin/hive-union/build/ql/scratchdir/hive_2011-07-28_13-48-37_450_4340421797588168189/-ext-10001 - file:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_04-06-57_227_5273764780066863405/-ext-10000 + file:/data/users/franklin/hive-union/build/ql/scratchdir/hive_2011-07-28_13-48-37_450_4340421797588168189/-ext-10000 @@ -333,7 +333,7 @@ - file:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_04-06-57_227_5273764780066863405/-ext-10001 + file:/data/users/franklin/hive-union/build/ql/scratchdir/hive_2011-07-28_13-48-37_450_4340421797588168189/-ext-10001 @@ -419,11 +419,11 @@ location - pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/src + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src transient_lastDdlTime - 1310382416 + 1311886115 @@ -481,11 +481,11 @@ location - pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/src + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src transient_lastDdlTime - 1310382416 + 1311886115 @@ -551,11 +551,11 @@ location - pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/src + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src transient_lastDdlTime - 1310382416 + 1311886115 @@ -613,11 +613,11 @@ location - pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/src + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src transient_lastDdlTime - 1310382416 + 1311886115 @@ -660,13 +660,13 @@ 1 - file:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_04-06-57_227_5273764780066863405/-ext-10001 + file:/data/users/franklin/hive-union/build/ql/scratchdir/hive_2011-07-28_13-48-37_450_4340421797588168189/-ext-10001 1 - file:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_04-06-57_227_5273764780066863405/-ext-10000/ + file:/data/users/franklin/hive-union/build/ql/scratchdir/hive_2011-07-28_13-48-37_450_4340421797588168189/-ext-10000/ @@ -679,21 +679,21 @@ - CNTR_NAME_FS_661_NUM_INPUT_ROWS + CNTR_NAME_FS_8_NUM_INPUT_ROWS - CNTR_NAME_FS_661_NUM_OUTPUT_ROWS + CNTR_NAME_FS_8_NUM_OUTPUT_ROWS - CNTR_NAME_FS_661_TIME_TAKEN + CNTR_NAME_FS_8_TIME_TAKEN - CNTR_NAME_FS_661_FATAL_ERROR + CNTR_NAME_FS_8_FATAL_ERROR - FS_661 + FS_8 @@ -771,21 +771,21 @@ - CNTR_NAME_SEL_660_NUM_INPUT_ROWS + CNTR_NAME_SEL_7_NUM_INPUT_ROWS - CNTR_NAME_SEL_660_NUM_OUTPUT_ROWS + CNTR_NAME_SEL_7_NUM_OUTPUT_ROWS - CNTR_NAME_SEL_660_TIME_TAKEN + CNTR_NAME_SEL_7_TIME_TAKEN - CNTR_NAME_SEL_660_FATAL_ERROR + CNTR_NAME_SEL_7_FATAL_ERROR - SEL_660 + SEL_7 @@ -844,21 +844,21 @@ - CNTR_NAME_UNION_659_NUM_INPUT_ROWS + CNTR_NAME_UNION_6_NUM_INPUT_ROWS - CNTR_NAME_UNION_659_NUM_OUTPUT_ROWS + CNTR_NAME_UNION_6_NUM_OUTPUT_ROWS - CNTR_NAME_UNION_659_TIME_TAKEN + CNTR_NAME_UNION_6_TIME_TAKEN - CNTR_NAME_UNION_659_FATAL_ERROR + CNTR_NAME_UNION_6_FATAL_ERROR - UNION_659 + UNION_6 @@ -936,21 +936,21 @@ - CNTR_NAME_SEL_658_NUM_INPUT_ROWS + CNTR_NAME_SEL_5_NUM_INPUT_ROWS - CNTR_NAME_SEL_658_NUM_OUTPUT_ROWS + CNTR_NAME_SEL_5_NUM_OUTPUT_ROWS - CNTR_NAME_SEL_658_TIME_TAKEN + CNTR_NAME_SEL_5_TIME_TAKEN - CNTR_NAME_SEL_658_FATAL_ERROR + CNTR_NAME_SEL_5_FATAL_ERROR - SEL_658 + SEL_5 @@ -1015,21 +1015,21 @@ - CNTR_NAME_FIL_663_NUM_INPUT_ROWS + CNTR_NAME_FIL_10_NUM_INPUT_ROWS - CNTR_NAME_FIL_663_NUM_OUTPUT_ROWS + CNTR_NAME_FIL_10_NUM_OUTPUT_ROWS - CNTR_NAME_FIL_663_TIME_TAKEN + CNTR_NAME_FIL_10_TIME_TAKEN - CNTR_NAME_FIL_663_FATAL_ERROR + CNTR_NAME_FIL_10_FATAL_ERROR - FIL_663 + FIL_10 @@ -1055,16 +1055,16 @@ - CNTR_NAME_TS_656_NUM_INPUT_ROWS + CNTR_NAME_TS_3_NUM_INPUT_ROWS - CNTR_NAME_TS_656_NUM_OUTPUT_ROWS + CNTR_NAME_TS_3_NUM_OUTPUT_ROWS - CNTR_NAME_TS_656_TIME_TAKEN + CNTR_NAME_TS_3_TIME_TAKEN - CNTR_NAME_TS_656_FATAL_ERROR + CNTR_NAME_TS_3_FATAL_ERROR @@ -1079,7 +1079,7 @@ - TS_656 + TS_3 @@ -1209,7 +1209,7 @@ - + _col0 @@ -1222,7 +1222,7 @@ - + _col1 @@ -1301,21 +1301,21 @@ - CNTR_NAME_SEL_655_NUM_INPUT_ROWS + CNTR_NAME_SEL_2_NUM_INPUT_ROWS - CNTR_NAME_SEL_655_NUM_OUTPUT_ROWS + CNTR_NAME_SEL_2_NUM_OUTPUT_ROWS - CNTR_NAME_SEL_655_TIME_TAKEN + CNTR_NAME_SEL_2_TIME_TAKEN - CNTR_NAME_SEL_655_FATAL_ERROR + CNTR_NAME_SEL_2_FATAL_ERROR - SEL_655 + SEL_2 @@ -1329,10 +1329,30 @@ - + + + _col0 + + + src + + + + + - + + + _col1 + + + src + + + + + @@ -1386,21 +1406,21 @@ - CNTR_NAME_FIL_662_NUM_INPUT_ROWS + CNTR_NAME_FIL_9_NUM_INPUT_ROWS - CNTR_NAME_FIL_662_NUM_OUTPUT_ROWS + CNTR_NAME_FIL_9_NUM_OUTPUT_ROWS - CNTR_NAME_FIL_662_TIME_TAKEN + CNTR_NAME_FIL_9_TIME_TAKEN - CNTR_NAME_FIL_662_FATAL_ERROR + CNTR_NAME_FIL_9_FATAL_ERROR - FIL_662 + FIL_9 @@ -1492,16 +1512,16 @@ - CNTR_NAME_TS_653_NUM_INPUT_ROWS + CNTR_NAME_TS_0_NUM_INPUT_ROWS - CNTR_NAME_TS_653_NUM_OUTPUT_ROWS + CNTR_NAME_TS_0_NUM_OUTPUT_ROWS - CNTR_NAME_TS_653_TIME_TAKEN + CNTR_NAME_TS_0_TIME_TAKEN - CNTR_NAME_TS_653_FATAL_ERROR + CNTR_NAME_TS_0_FATAL_ERROR @@ -1516,7 +1536,7 @@ - TS_653 + TS_0 @@ -1539,7 +1559,7 @@ - pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/src + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src null-subquery1:unioninput-subquery1:src @@ -1554,7 +1574,7 @@ - pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/src + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src src @@ -1611,11 +1631,11 @@ location - pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/src + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src transient_lastDdlTime - 1310382416 + 1311886115 @@ -1673,11 +1693,11 @@ location - pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/src + pfile:/data/users/franklin/hive-union/build/ql/test/data/warehouse/src transient_lastDdlTime - 1310382416 + 1311886115 Index: ql/src/test/queries/clientpositive/union24.q =================================================================== --- ql/src/test/queries/clientpositive/union24.q (revision 0) +++ ql/src/test/queries/clientpositive/union24.q (revision 0) @@ -0,0 +1,42 @@ + +explain extended + select * from ( + select 1234 from src limit 10 + union all + select '1234' from src limit 10 + ) t; + +select * from ( + select 1234 from src limit 10 + union all + select '1234' from src limit 10 +) t; + + +explain extended + select * from ( + select 1234, 'some-string', cast(2345 as bigint) from src limit 10 + union all + select 'other-string', 1, 0 from src limit 10 + ) t; + +select * from ( + select 1234, 'some-string', cast(2345 as bigint) from src limit 10 + union all + select 'other-string', 1, 0 from src limit 10 +) t; + + +explain extended + select * from ( + select transform(*) using 'tr \t _' as (c) from src limit 10 + union all + select 12345 as c from src limit 10 + ) t; + +select * from ( + select transform(*) using 'tr \t _' as (c) from src limit 10 + union all + select 12345 as c from src limit 10 +) t; + Index: ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (revision 1151937) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (working copy) @@ -343,6 +343,15 @@ opCtxMap.get(inp).rowObjectInspector = this.rowObjectInspector; } } + } else if (op instanceof UnionOperator) { + if (op != null) { + if (opCtxMap != null) { + MapOpCtx m = opCtxMap.get(inp); + if (m != null) { + op.inputObjInspectors[0] = m.getRowObjectInspector(); + } + } + } } } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java (revision 1151937) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java (working copy) @@ -132,5 +132,9 @@ this.isHiddenVirtualCol = isHiddenVirtualCol; } + public ColumnInfo clone() { + return new ColumnInfo(internalName, type, tabAlias, isVirtualCol, + isHiddenVirtualCol); + } } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (revision 1151937) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (working copy) @@ -586,8 +586,26 @@ registerNumericType(Constants.STRING_TYPE_NAME, 7); } + /** * Find a common class that objects of both TypeInfo a and TypeInfo b can + * convert to. This is used when unioning objects of type a and b. + * + * @return null if no common class could be found + */ + public static TypeInfo getCommonClassForUnion(TypeInfo a, TypeInfo b) { + if (a.equals(b)) { + return a; + } + TypeInfo t = getCommonClass(a, b); + if (t == null) { + t = getCommonClassForComparison(a, b); + } + return t; + } + + /** + * Find a common class that objects of both TypeInfo a and TypeInfo b can * convert to. This is used for comparing objects of type a and type b. * * When we are comparing string and double, we will always convert both of Index: ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java (revision 1151937) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java (working copy) @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; /** * Union Operator Just forwards. Doesn't do anything itself. @@ -59,11 +60,8 @@ int parents = parentOperators.size(); parentObjInspectors = new StructObjectInspector[parents]; parentFields = new List[parents]; - for (int p = 0; p < parents; p++) { - parentObjInspectors[p] = (StructObjectInspector) inputObjInspectors[p]; - parentFields[p] = parentObjInspectors[p].getAllStructFieldRefs(); - } - + updateInspectors(); + // Get columnNames from the first parent int columns = parentFields[0].size(); ArrayList columnNames = new ArrayList(columns); @@ -76,13 +74,11 @@ for (int c = 0; c < columns; c++) { columnTypeResolvers[c] = new ReturnObjectInspectorResolver(true); } - - for (int p = 0; p < parents; p++) { - assert (parentFields[p].size() == columns); - for (int c = 0; c < columns; c++) { - columnTypeResolvers[c].update(parentFields[p].get(c) - .getFieldObjectInspector()); - } + + List colInfos = this.getSchema().getSignature(); + for (int i = 0; i < colInfos.size(); i++) { + columnTypeResolvers[i].update(TypeInfoUtils + .getStandardWritableObjectInspectorFromTypeInfo(colInfos.get(i).getType())); } ArrayList outputFieldOIs = new ArrayList( @@ -131,6 +127,19 @@ forward(row, inputObjInspectors[tag]); } } + + public void cleanUpInputFileChanged() throws HiveException { + updateInspectors(); + super.cleanUpInputFileChanged(); + } + + private void updateInspectors() { + int parents = parentOperators.size(); + for (int p = 0; p < parents; p++) { + parentObjInspectors[p] = (StructObjectInspector) inputObjInspectors[p]; + parentFields[p] = parentObjInspectors[p].getAllStructFieldRefs(); + } + } /** * @return the name of the operator Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1151937) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -5954,8 +5954,9 @@ + ". Column positions should match for a UNION")); } //try widening coversion, otherwise fail union - TypeInfo commonTypeInfo = FunctionRegistry.getCommonClassForComparison(lInfo.getType(), - rInfo.getType()); + TypeInfo commonTypeInfo = FunctionRegistry.getCommonClassForUnion( + lInfo.getType(), rInfo.getType()); + if (commonTypeInfo == null) { throw new SemanticException(generateErrorMessage(tabref, "Schema of both sides of union should match: Column " + field @@ -5971,9 +5972,10 @@ String field = lEntry.getKey(); ColumnInfo lInfo = lEntry.getValue(); ColumnInfo rInfo = rightmap.get(field); - lInfo.setType(FunctionRegistry.getCommonClassForComparison(lInfo.getType(), - rInfo.getType())); - unionoutRR.put(unionalias, field, lInfo); + ColumnInfo commonInfo = lInfo.clone(); + commonInfo.setType(FunctionRegistry.getCommonClassForUnion( + lInfo.getType(), rInfo.getType())); + unionoutRR.put(unionalias, field, commonInfo); } // If one of the children is a union, merge with it