Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (revision 958668) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (working copy) @@ -290,27 +290,29 @@ // which should be fixed before remove this if ((child instanceof FileSinkOperator) || (child instanceof ScriptOperator) - || (child instanceof UDTFOperator) || (child instanceof LimitOperator) || (child instanceof UnionOperator)) { cppCtx.getPrunedColLists() .put(op, cppCtx.getColsFromSelectExpr(op)); return null; + } else if ((child instanceof UDTFOperator)) { + cppCtx.getPrunedColLists().remove(op); + return null; } } } - cols = cppCtx.genColLists(op); SelectDesc conf = op.getConf(); + if (conf.isSelStarNoCompute()) { + return null; + } + + cols = cppCtx.genColLists(op); // The input to the select does not matter. Go over the expressions // and return the ones which have a marked column cppCtx.getPrunedColLists().put(op, cppCtx.getSelectColsFromChildren(op, cols)); - if (conf.isSelStarNoCompute()) { - return null; - } - // do we need to prune the select operator? List originalColList = op.getConf().getColList(); List columns = new ArrayList(); Index: ql/src/test/queries/clientpositive/lateral_view.q =================================================================== --- ql/src/test/queries/clientpositive/lateral_view.q (revision 959361) +++ ql/src/test/queries/clientpositive/lateral_view.q (working copy) @@ -1,3 +1,6 @@ +drop table tmp_pyang_lv; +drop table tmp_pyang_src_rcfile; + create table tmp_pyang_lv (inputs string) stored as rcfile; insert overwrite table tmp_pyang_lv select key from src; @@ -20,4 +23,13 @@ select myCol from tmp_pyang_lv LATERAL VIEW explode(array(1,2,3)) myTab as myCol limit 3; + +create table tmp_pyang_src_rcfile (key string, value array) stored as rcfile; +insert overwrite table tmp_pyang_src_rcfile select key, array(value) from src limit 20; + +select key,value from tmp_pyang_src_rcfile lateral view explode(value) myCol AS myTable; +select myTable from tmp_pyang_src_rcfile lateral view explode(value) myCol AS myTable; +select * from tmp_pyang_src_rcfile lateral view explode(value) myCol AS myTable; + +drop table tmp_pyang_src_rcfile; drop table tmp_pyang_lv; Index: ql/src/test/results/clientpositive/lateral_view.q.out =================================================================== --- ql/src/test/results/clientpositive/lateral_view.q.out (revision 959361) +++ ql/src/test/results/clientpositive/lateral_view.q.out (working copy) @@ -1,3 +1,11 @@ +PREHOOK: query: drop table tmp_pyang_lv +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tmp_pyang_lv +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table tmp_pyang_src_rcfile +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tmp_pyang_src_rcfile +POSTHOOK: type: DROPTABLE PREHOOK: query: create table tmp_pyang_lv (inputs string) stored as rcfile PREHOOK: type: CREATETABLE POSTHOOK: query: create table tmp_pyang_lv (inputs string) stored as rcfile @@ -106,7 +114,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-18_19-54-16_286_7545304927031880480/10002 + file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-56-34_771_1384394814853513597/10002 Reduce Output Operator key expressions: expr: _col0 @@ -435,24 +443,24 @@ SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol SORT BY key ASC, myCol ASC LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-18_19-54-16_764_354722541681050276/10000 +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-56-35_280_2351098561574516560/10000 POSTHOOK: query: -- Verify that * selects columns from both tables SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol SORT BY key ASC, myCol ASC LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-18_19-54-16_764_354722541681050276/10000 +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-56-35_280_2351098561574516560/10000 POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] 0 val_0 1 PREHOOK: query: -- TABLE.* should be supported SELECT myTable.* FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol LIMIT 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-18_19-54-23_656_7186983670906657556/10000 +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-56-46_394_6431173834698620357/10000 POSTHOOK: query: -- TABLE.* should be supported SELECT myTable.* FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol LIMIT 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-18_19-54-23_656_7186983670906657556/10000 +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-56-46_394_6431173834698620357/10000 POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] 1 2 @@ -461,12 +469,12 @@ SELECT myTable.myCol, myTable2.myCol2 FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol LATERAL VIEW explode(array('a', 'b', 'c')) myTable2 AS myCol2 LIMIT 9 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-18_19-54-27_321_7329581422596695757/10000 +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-56-50_054_5409952490144894399/10000 POSTHOOK: query: -- Multiple lateral views should result in a Cartesian product SELECT myTable.myCol, myTable2.myCol2 FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol LATERAL VIEW explode(array('a', 'b', 'c')) myTable2 AS myCol2 LIMIT 9 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-18_19-54-27_321_7329581422596695757/10000 +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-56-50_054_5409952490144894399/10000 POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] 1 a 1 b @@ -481,12 +489,12 @@ SELECT myTable2.* FROM src LATERAL VIEW explode(array(array(1,2,3))) myTable AS myCol LATERAL VIEW explode(myTable.myCol) myTable2 AS myCol2 LIMIT 3 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-18_19-54-31_056_3810278535508636145/10000 +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-56-53_866_5115169352556230824/10000 POSTHOOK: query: -- Should be able to reference tables generated earlier SELECT myTable2.* FROM src LATERAL VIEW explode(array(array(1,2,3))) myTable AS myCol LATERAL VIEW explode(myTable.myCol) myTable2 AS myCol2 LIMIT 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-18_19-54-31_056_3810278535508636145/10000 +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-56-53_866_5115169352556230824/10000 POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] 1 2 @@ -558,18 +566,138 @@ PREHOOK: query: select myCol from tmp_pyang_lv LATERAL VIEW explode(array(1,2,3)) myTab as myCol limit 3 PREHOOK: type: QUERY PREHOOK: Input: default@tmp_pyang_lv -PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-18_19-54-35_530_6647775029322914068/10000 +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-56-57_350_1041686738361607390/10000 POSTHOOK: query: select myCol from tmp_pyang_lv LATERAL VIEW explode(array(1,2,3)) myTab as myCol limit 3 POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp_pyang_lv -POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-18_19-54-35_530_6647775029322914068/10000 +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-56-57_350_1041686738361607390/10000 POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] 1 2 3 +PREHOOK: query: create table tmp_pyang_src_rcfile (key string, value array) stored as rcfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table tmp_pyang_src_rcfile (key string, value array) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tmp_pyang_src_rcfile +POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tmp_pyang_src_rcfile select key, array(value) from src limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tmp_pyang_src_rcfile +POSTHOOK: query: insert overwrite table tmp_pyang_src_rcfile select key, array(value) from src limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tmp_pyang_src_rcfile +POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select key,value from tmp_pyang_src_rcfile lateral view explode(value) myCol AS myTable +PREHOOK: type: QUERY +PREHOOK: Input: default@tmp_pyang_src_rcfile +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-57-06_754_8866697006517011441/10000 +POSTHOOK: query: select key,value from tmp_pyang_src_rcfile lateral view explode(value) myCol AS myTable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmp_pyang_src_rcfile +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-57-06_754_8866697006517011441/10000 +POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 ["val_238"] +86 ["val_86"] +311 ["val_311"] +27 ["val_27"] +165 ["val_165"] +409 ["val_409"] +255 ["val_255"] +278 ["val_278"] +98 ["val_98"] +484 ["val_484"] +265 ["val_265"] +193 ["val_193"] +401 ["val_401"] +150 ["val_150"] +273 ["val_273"] +224 ["val_224"] +369 ["val_369"] +66 ["val_66"] +128 ["val_128"] +213 ["val_213"] +PREHOOK: query: select myTable from tmp_pyang_src_rcfile lateral view explode(value) myCol AS myTable +PREHOOK: type: QUERY +PREHOOK: Input: default@tmp_pyang_src_rcfile +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-57-10_636_7906165775405384042/10000 +POSTHOOK: query: select myTable from tmp_pyang_src_rcfile lateral view explode(value) myCol AS myTable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmp_pyang_src_rcfile +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-57-10_636_7906165775405384042/10000 +POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +val_238 +val_86 +val_311 +val_27 +val_165 +val_409 +val_255 +val_278 +val_98 +val_484 +val_265 +val_193 +val_401 +val_150 +val_273 +val_224 +val_369 +val_66 +val_128 +val_213 +PREHOOK: query: select * from tmp_pyang_src_rcfile lateral view explode(value) myCol AS myTable +PREHOOK: type: QUERY +PREHOOK: Input: default@tmp_pyang_src_rcfile +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-57-13_849_7444595580149157006/10000 +POSTHOOK: query: select * from tmp_pyang_src_rcfile lateral view explode(value) myCol AS myTable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmp_pyang_src_rcfile +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-07-08_21-57-13_849_7444595580149157006/10000 +POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 ["val_238"] val_238 +86 ["val_86"] val_86 +311 ["val_311"] val_311 +27 ["val_27"] val_27 +165 ["val_165"] val_165 +409 ["val_409"] val_409 +255 ["val_255"] val_255 +278 ["val_278"] val_278 +98 ["val_98"] val_98 +484 ["val_484"] val_484 +265 ["val_265"] val_265 +193 ["val_193"] val_193 +401 ["val_401"] val_401 +150 ["val_150"] val_150 +273 ["val_273"] val_273 +224 ["val_224"] val_224 +369 ["val_369"] val_369 +66 ["val_66"] val_66 +128 ["val_128"] val_128 +213 ["val_213"] val_213 +PREHOOK: query: drop table tmp_pyang_src_rcfile +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tmp_pyang_src_rcfile +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@tmp_pyang_src_rcfile +POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: drop table tmp_pyang_lv PREHOOK: type: DROPTABLE POSTHOOK: query: drop table tmp_pyang_lv POSTHOOK: type: DROPTABLE POSTHOOK: Output: default@tmp_pyang_lv POSTHOOK: Lineage: tmp_pyang_lv.inputs SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmp_pyang_src_rcfile.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]