diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index aab3aab..697a2fc 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -1549,7 +1549,8 @@ private void maskPatterns(Pattern[] patterns, String fname) throws Exception { ".*Input:.*/data/files/.*", ".*Output:.*/data/files/.*", ".*total number of created files now is.*", - ".*.hive-staging.*" + ".*.hive-staging.*", + ".*org\\.apache\\.hadoop\\.hive\\.contrib\\.udtf\\.example\\.GenericUDTFCount2.*" }); private final Pattern[] partialReservedPlanMask = toPattern(new String[] { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java index 3859177..6f66bec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.exec.UDTFOperator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.io.ContentSummaryInputFormat; @@ -283,7 +284,7 @@ private boolean isConvertible(FetchData fetch) { private boolean isConvertible(FetchData fetch, Operator operator, Set> traversed) { if (operator instanceof ReduceSinkOperator || operator instanceof CommonJoinOperator - || operator instanceof ScriptOperator) { + || operator instanceof ScriptOperator || operator instanceof UDTFOperator) { return false; } diff --git a/ql/src/test/queries/clientpositive/udtf_nofetchtask.q b/ql/src/test/queries/clientpositive/udtf_nofetchtask.q new file mode 100644 index 0000000..d4cfee4 --- /dev/null +++ b/ql/src/test/queries/clientpositive/udtf_nofetchtask.q @@ -0,0 +1,16 @@ +create temporary function udtfCount2 as 'org.apache.hadoop.hive.contrib.udtf.example.GenericUDTFCount2'; + +set hive.fetch.task.conversion=minimal; +-- Correct output should be 2 rows +explain +select udtfCount2() from src; + +select udtfCount2() from src; + +set hive.fetch.task.conversion=more; +-- Make sure we do not use fetch task conversion +explain +select udtfCount2() from src; + +select udtfCount2() from src; + diff --git a/ql/src/test/results/clientpositive/udtf_nofetchtask.q.out b/ql/src/test/results/clientpositive/udtf_nofetchtask.q.out new file mode 100644 index 0000000..688b673 --- /dev/null +++ b/ql/src/test/results/clientpositive/udtf_nofetchtask.q.out @@ -0,0 +1,102 @@ +#### A masked pattern was here #### +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: udtfcount2 +#### A masked pattern was here #### +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: udtfcount2 +PREHOOK: query: -- Correct output should be 2 rows +explain +select udtfCount2() from src +PREHOOK: type: QUERY +POSTHOOK: query: -- Correct output should be 2 rows +explain +select udtfCount2() from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE +#### A masked pattern was here #### + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select udtfCount2() from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select udtfCount2() from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +500 +500 +PREHOOK: query: -- Make sure we do not use fetch task conversion +explain +select udtfCount2() from src +PREHOOK: type: QUERY +POSTHOOK: query: -- Make sure we do not use fetch task conversion +explain +select udtfCount2() from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE +#### A masked pattern was here #### + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select udtfCount2() from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select udtfCount2() from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +500 +500