Index: ql/src/test/results/clientpositive/ppd_union_view.q.out =================================================================== --- ql/src/test/results/clientpositive/ppd_union_view.q.out (revision 0) +++ ql/src/test/results/clientpositive/ppd_union_view.q.out (revision 0) @@ -0,0 +1,954 @@ +PREHOOK: query: -- test predicate pushdown on a view with a union + +drop view v +PREHOOK: type: DROPVIEW +POSTHOOK: query: -- test predicate pushdown on a view with a union + +drop view v +POSTHOOK: type: DROPVIEW +PREHOOK: query: create table t1_new (key string, value string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table t1_new (key string, value string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@t1_new +PREHOOK: query: insert overwrite table t1_new partition (ds = '2011-10-15') +select 'key1', 'value1' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t1_new@ds=2011-10-15 +POSTHOOK: query: insert overwrite table t1_new partition (ds = '2011-10-15') +select 'key1', 'value1' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t1_new@ds=2011-10-15 +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +PREHOOK: query: insert overwrite table t1_new partition (ds = '2011-10-16') +select 'key2', 'value2' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t1_new@ds=2011-10-16 +POSTHOOK: query: insert overwrite table t1_new partition (ds = '2011-10-16') +select 'key2', 'value2' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t1_new@ds=2011-10-16 +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +PREHOOK: query: create table t1_old (keymap string, value string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table t1_old (keymap string, value string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@t1_old +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +PREHOOK: query: insert overwrite table t1_old partition (ds = '2011-10-13') +select 'keymap3', 'value3' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t1_old@ds=2011-10-13 +POSTHOOK: query: insert overwrite table t1_old partition (ds = '2011-10-13') +select 'keymap3', 'value3' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t1_old@ds=2011-10-13 +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +PREHOOK: query: insert overwrite table t1_old partition (ds = '2011-10-14') +select 'keymap4', 'value4' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t1_old@ds=2011-10-14 +POSTHOOK: query: insert overwrite table t1_old partition (ds = '2011-10-14') +select 'keymap4', 'value4' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t1_old@ds=2011-10-14 +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).value SIMPLE [] +PREHOOK: query: create table t1_mapping (key string, keymap string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table t1_mapping (key string, keymap string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@t1_mapping +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).value SIMPLE [] +PREHOOK: query: insert overwrite table t1_mapping partition (ds = '2011-10-13') +select 'key3', 'keymap3' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t1_mapping@ds=2011-10-13 +POSTHOOK: query: insert overwrite table t1_mapping partition (ds = '2011-10-13') +select 'key3', 'keymap3' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t1_mapping@ds=2011-10-13 +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).value SIMPLE [] +PREHOOK: query: insert overwrite table t1_mapping partition (ds = '2011-10-14') +select 'key4', 'keymap4' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@t1_mapping@ds=2011-10-14 +POSTHOOK: query: insert overwrite table t1_mapping partition (ds = '2011-10-14') +select 'key4', 'keymap4' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@t1_mapping@ds=2011-10-14 +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).value SIMPLE [] +PREHOOK: query: create view t1 partitioned on (ds) as +select * from +( +select key, value, ds from t1_new +union all +select key, value, t1_old.ds from t1_old join t1_mapping +on t1_old.keymap = t1_mapping.keymap and + t1_old.ds = t1_mapping.ds +) subq +PREHOOK: type: CREATEVIEW +PREHOOK: Output: file:/tmp/njain/hive_2011-10-19_12-06-05_776_5819278510791879197/-mr-10000 +POSTHOOK: query: create view t1 partitioned on (ds) as +select * from +( +select key, value, ds from t1_new +union all +select key, value, t1_old.ds from t1_old join t1_mapping +on t1_old.keymap = t1_mapping.keymap and + t1_old.ds = t1_mapping.ds +) subq +POSTHOOK: type: CREATEVIEW +POSTHOOK: Output: default@t1 +POSTHOOK: Output: file:/tmp/njain/hive_2011-10-19_12-06-05_776_5819278510791879197/-mr-10000 +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).value SIMPLE [] +PREHOOK: query: explain extended +select * from t1 where ds = '2011-10-13' +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select * from t1 where ds = '2011-10-13' +POSTHOOK: type: QUERY +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).value SIMPLE [] +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME t1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2011-10-13')))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-4 + Stage-4 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + t1-subquery2:subq-subquery2:t1_mapping + TableScan + alias: t1_mapping + GatherStats: false + Reduce Output Operator + key expressions: + expr: keymap + type: string + expr: ds + type: string + sort order: ++ + Map-reduce partition columns: + expr: keymap + type: string + expr: ds + type: string + tag: 1 + value expressions: + expr: key + type: string + t1-subquery2:subq-subquery2:t1_old + TableScan + alias: t1_old + GatherStats: false + Reduce Output Operator + key expressions: + expr: keymap + type: string + expr: ds + type: string + sort order: ++ + Map-reduce partition columns: + expr: keymap + type: string + expr: ds + type: string + tag: 0 + value expressions: + expr: value + type: string + expr: ds + type: string + Needs Tagging: true + Path -> Alias: + pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_mapping/ds=2011-10-13 [t1-subquery2:subq-subquery2:t1_mapping] + pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_old/ds=2011-10-13 [t1-subquery2:subq-subquery2:t1_old] + Path -> Partition: + pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_mapping/ds=2011-10-13 + Partition + base file name: ds=2011-10-13 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2011-10-13 + properties: + bucket_count -1 + columns key,keymap + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_mapping/ds=2011-10-13 + name default.t1_mapping + numFiles 1 + numPartitions 2 + numRows 1 + partition_columns ds + rawDataSize 12 + serialization.ddl struct t1_mapping { string key, string keymap} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 13 + transient_lastDdlTime 1319051160 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,keymap + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_mapping + name default.t1_mapping + numFiles 2 + numPartitions 2 + numRows 2 + partition_columns ds + rawDataSize 24 + serialization.ddl struct t1_mapping { string key, string keymap} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 26 + transient_lastDdlTime 1319051165 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t1_mapping + name: default.t1_mapping + pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_old/ds=2011-10-13 + Partition + base file name: ds=2011-10-13 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2011-10-13 + properties: + bucket_count -1 + columns keymap,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_old/ds=2011-10-13 + name default.t1_old + numFiles 1 + numPartitions 2 + numRows 1 + partition_columns ds + rawDataSize 14 + serialization.ddl struct t1_old { string keymap, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 15 + transient_lastDdlTime 1319051151 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns keymap,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_old + name default.t1_old + numFiles 2 + numPartitions 2 + numRows 2 + partition_columns ds + rawDataSize 28 + serialization.ddl struct t1_old { string keymap, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 30 + transient_lastDdlTime 1319051156 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t1_old + name: default.t1_old + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col1} {VALUE._col2} + 1 {VALUE._col0} + handleSkewJoin: false + outputColumnNames: _col1, _col2, _col5 + Select Operator + expressions: + expr: _col5 + type: string + expr: _col1 + type: string + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-mr-10002 + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-mr-10002 + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-ext-10001 + NumFilesPerFileSink: 1 + Stats Publishing Key Prefix: file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-ext-10001/ + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string:string:string + escape.delim \ + serialization.format 1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-mr-10003 + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-ext-10001 + NumFilesPerFileSink: 1 + Stats Publishing Key Prefix: file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-ext-10001/ + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string:string:string + escape.delim \ + serialization.format 1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: + file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-mr-10002 [file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-mr-10002] + file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-mr-10003 [file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-mr-10003] + Path -> Partition: + file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-mr-10002 + Partition + base file name: -mr-10002 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-mr-10003 + Partition + base file name: -mr-10003 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: + t1-subquery1:subq-subquery1:t1_new + TableScan + alias: t1_new + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: + expr: (ds = '2011-10-13') + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + expr: ds + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/njain/hive_2011-10-19_12-06-05_880_7698165420419253512/-mr-10003 + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: select * from t1 where ds = '2011-10-13' +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_mapping@ds=2011-10-13 +PREHOOK: Input: default@t1_old@ds=2011-10-13 +PREHOOK: Output: file:/tmp/njain/hive_2011-10-19_12-06-06_192_354554347940334190/-mr-10000 +POSTHOOK: query: select * from t1 where ds = '2011-10-13' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_mapping@ds=2011-10-13 +POSTHOOK: Input: default@t1_old@ds=2011-10-13 +POSTHOOK: Output: file:/tmp/njain/hive_2011-10-19_12-06-06_192_354554347940334190/-mr-10000 +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).value SIMPLE [] +key3 value3 2011-10-13 +PREHOOK: query: select * from t1 where ds = '2011-10-14' +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_mapping@ds=2011-10-14 +PREHOOK: Input: default@t1_old@ds=2011-10-14 +PREHOOK: Output: file:/tmp/njain/hive_2011-10-19_12-06-16_134_8398650711346281605/-mr-10000 +POSTHOOK: query: select * from t1 where ds = '2011-10-14' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_mapping@ds=2011-10-14 +POSTHOOK: Input: default@t1_old@ds=2011-10-14 +POSTHOOK: Output: file:/tmp/njain/hive_2011-10-19_12-06-16_134_8398650711346281605/-mr-10000 +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).value SIMPLE [] +key4 value4 2011-10-14 +PREHOOK: query: explain extended +select * from t1 where ds = '2011-10-15' +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select * from t1 where ds = '2011-10-15' +POSTHOOK: type: QUERY +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).value SIMPLE [] +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME t1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2011-10-15')))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-4 + Stage-4 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + t1-subquery2:subq-subquery2:t1_mapping + TableScan + alias: t1_mapping + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: + expr: (ds = '2011-10-15') + type: boolean + Reduce Output Operator + key expressions: + expr: keymap + type: string + expr: ds + type: string + sort order: ++ + Map-reduce partition columns: + expr: keymap + type: string + expr: ds + type: string + tag: 1 + value expressions: + expr: key + type: string + t1-subquery2:subq-subquery2:t1_old + TableScan + alias: t1_old + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: + expr: (ds = '2011-10-15') + type: boolean + Reduce Output Operator + key expressions: + expr: keymap + type: string + expr: ds + type: string + sort order: ++ + Map-reduce partition columns: + expr: keymap + type: string + expr: ds + type: string + tag: 0 + value expressions: + expr: value + type: string + expr: ds + type: string + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col1} {VALUE._col2} + 1 {VALUE._col0} + handleSkewJoin: false + outputColumnNames: _col1, _col2, _col5 + Select Operator + expressions: + expr: _col5 + type: string + expr: _col1 + type: string + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-mr-10002 + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-mr-10002 + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-ext-10001 + NumFilesPerFileSink: 1 + Stats Publishing Key Prefix: file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-ext-10001/ + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string:string:string + escape.delim \ + serialization.format 1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-mr-10003 + Union + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: _col0 + type: string + expr: _col1 + type: string + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-ext-10001 + NumFilesPerFileSink: 1 + Stats Publishing Key Prefix: file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-ext-10001/ + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string:string:string + escape.delim \ + serialization.format 1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: + file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-mr-10002 [file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-mr-10002] + file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-mr-10003 [file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-mr-10003] + Path -> Partition: + file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-mr-10002 + Partition + base file name: -mr-10002 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-mr-10003 + Partition + base file name: -mr-10003 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + + Stage: Stage-4 + Map Reduce + Alias -> Map Operator Tree: + t1-subquery1:subq-subquery1:t1_new + TableScan + alias: t1_new + GatherStats: false + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + expr: ds + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/tmp/njain/hive_2011-10-19_12-06-25_964_2220398768855201359/-mr-10003 + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Needs Tagging: false + Path -> Alias: + pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_new/ds=2011-10-15 [t1-subquery1:subq-subquery1:t1_new] + Path -> Partition: + pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_new/ds=2011-10-15 + Partition + base file name: ds=2011-10-15 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2011-10-15 + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_new/ds=2011-10-15 + name default.t1_new + numFiles 1 + numPartitions 2 + numRows 1 + partition_columns ds + rawDataSize 11 + serialization.ddl struct t1_new { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 12 + transient_lastDdlTime 1319051142 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.types string:string + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location pfile:/data/users/njain/hive2/build/ql/test/data/warehouse/t1_new + name default.t1_new + numFiles 2 + numPartitions 2 + numRows 2 + partition_columns ds + rawDataSize 22 + serialization.ddl struct t1_new { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 24 + transient_lastDdlTime 1319051146 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.t1_new + name: default.t1_new + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: select * from t1 where ds = '2011-10-15' +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_new@ds=2011-10-15 +PREHOOK: Output: file:/tmp/njain/hive_2011-10-19_12-06-26_193_8893872878286188124/-mr-10000 +POSTHOOK: query: select * from t1 where ds = '2011-10-15' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_new@ds=2011-10-15 +POSTHOOK: Output: file:/tmp/njain/hive_2011-10-19_12-06-26_193_8893872878286188124/-mr-10000 +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).value SIMPLE [] +key1 value1 2011-10-15 +PREHOOK: query: select * from t1 where ds = '2011-10-16' +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_new@ds=2011-10-16 +PREHOOK: Output: file:/tmp/njain/hive_2011-10-19_12-06-35_704_607427779567049424/-mr-10000 +POSTHOOK: query: select * from t1 where ds = '2011-10-16' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_new@ds=2011-10-16 +POSTHOOK: Output: file:/tmp/njain/hive_2011-10-19_12-06-35_704_607427779567049424/-mr-10000 +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).key SIMPLE [] +POSTHOOK: Lineage: t1_mapping PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-15).value SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).key SIMPLE [] +POSTHOOK: Lineage: t1_new PARTITION(ds=2011-10-16).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-13).value SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).keymap SIMPLE [] +POSTHOOK: Lineage: t1_old PARTITION(ds=2011-10-14).value SIMPLE [] +key2 value2 2011-10-16 Index: ql/src/test/queries/clientpositive/ppd_union_view.q =================================================================== --- ql/src/test/queries/clientpositive/ppd_union_view.q (revision 0) +++ ql/src/test/queries/clientpositive/ppd_union_view.q (revision 0) @@ -0,0 +1,51 @@ +-- test predicate pushdown on a view with a union + +drop view v; + +create table t1_new (key string, value string) partitioned by (ds string); + +insert overwrite table t1_new partition (ds = '2011-10-15') +select 'key1', 'value1' from src limit 1; + +insert overwrite table t1_new partition (ds = '2011-10-16') +select 'key2', 'value2' from src limit 1; + +create table t1_old (keymap string, value string) partitioned by (ds string); + +insert overwrite table t1_old partition (ds = '2011-10-13') +select 'keymap3', 'value3' from src limit 1; + +insert overwrite table t1_old partition (ds = '2011-10-14') +select 'keymap4', 'value4' from src limit 1; + +create table t1_mapping (key string, keymap string) partitioned by (ds string); + +insert overwrite table t1_mapping partition (ds = '2011-10-13') +select 'key3', 'keymap3' from src limit 1; + +insert overwrite table t1_mapping partition (ds = '2011-10-14') +select 'key4', 'keymap4' from src limit 1; + + +create view t1 partitioned on (ds) as +select * from +( +select key, value, ds from t1_new +union all +select key, value, t1_old.ds from t1_old join t1_mapping +on t1_old.keymap = t1_mapping.keymap and + t1_old.ds = t1_mapping.ds +) subq; + +explain extended +select * from t1 where ds = '2011-10-13'; + +select * from t1 where ds = '2011-10-13'; + +select * from t1 where ds = '2011-10-14'; + +explain extended +select * from t1 where ds = '2011-10-15'; + +select * from t1 where ds = '2011-10-15'; +select * from t1 where ds = '2011-10-16'; \ No newline at end of file