diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 08de4a7..e207942 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -762,6 +762,7 @@ minillaplocal.query.files=\ vector_reuse_scratchcols.q,\ vector_string_decimal.q,\ vector_udf_adaptor_1.q,\ + vector_udf_inline.q,\ vector_udf_string_to_boolean.q,\ vector_udf1.q,\ vector_udf2.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index 93212ce..e7f2b54 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -3218,6 +3218,12 @@ private VectorExpression getCustomUDFExpression(ExprNodeGenericFuncDesc expr, Ve argDescs[i].setVariable(getInputColumnIndex(((ExprNodeColumnDesc) child).getColumn())); } else if (child instanceof ExprNodeConstantDesc) { // this is a constant (or null) + if (child.getTypeInfo().getCategory() != Category.PRIMITIVE) { + + // Complex type constants currently not supported by VectorUDFArgDesc.prepareConstant. + throw new HiveException( + "Unable to vectorize custom UDF. Complex type constants not supported: " + child); + } argDescs[i].setConstant((ExprNodeConstantDesc) child); } else if (child instanceof ExprNodeDynamicValueDesc) { VectorExpression e = getVectorExpression(child, VectorExpressionDescriptor.Mode.PROJECTION); diff --git ql/src/test/queries/clientpositive/udf_context_aware.q ql/src/test/queries/clientpositive/udf_context_aware.q index 04cb512..b12c213 100644 --- ql/src/test/queries/clientpositive/udf_context_aware.q +++ ql/src/test/queries/clientpositive/udf_context_aware.q @@ -1,3 +1,5 @@ +SET hive.vectorized.execution.enabled=false; + create temporary function counter as 'org.apache.hadoop.hive.ql.udf.generic.DummyContextUDF'; set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; diff --git ql/src/test/queries/clientpositive/udf_inline.q ql/src/test/queries/clientpositive/udf_inline.q index 95d55f7..111e452 100644 --- ql/src/test/queries/clientpositive/udf_inline.q +++ ql/src/test/queries/clientpositive/udf_inline.q @@ -1,4 +1,5 @@ set hive.fetch.task.conversion=more; +SET hive.vectorized.execution.enabled=false; describe function inline; diff --git ql/src/test/queries/clientpositive/vector_udf_inline.q ql/src/test/queries/clientpositive/vector_udf_inline.q new file mode 100644 index 0000000..5e073500 --- /dev/null +++ ql/src/test/queries/clientpositive/vector_udf_inline.q @@ -0,0 +1,32 @@ +set hive.fetch.task.conversion=none; +SET hive.vectorized.execution.enabled=true; + +-- HIVE-19024: Vectorization: Disable complex type constants for VectorUDFAdaptor + +describe function inline; + +explain vectorization expression +SELECT inline( + ARRAY( + STRUCT (1,'dude!'), + STRUCT (2,'Wheres'), + STRUCT (3,'my car?') + ) +) as (id, text) FROM SRC limit 2; + +SELECT inline( + ARRAY( + STRUCT (1,'dude!'), + STRUCT (2,'Wheres'), + STRUCT (3,'my car?') + ) +) as (id, text) FROM SRC limit 2; + +-- HIVE-3475 INLINE UDTF doesn't convert types properly +select * from (SELECT + ARRAY( + STRUCT (1,'dude!'), + STRUCT (2,'Wheres'), + STRUCT (3,'my car?') + ) as value FROM SRC limit 1) input + LATERAL VIEW inline(value) myTable AS id, text; diff --git ql/src/test/results/clientpositive/llap/vector_udf_inline.q.out ql/src/test/results/clientpositive/llap/vector_udf_inline.q.out new file mode 100644 index 0000000..67e8c91 --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_udf_inline.q.out @@ -0,0 +1,122 @@ +PREHOOK: query: describe function inline +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: describe function inline +POSTHOOK: type: DESCFUNCTION +inline( ARRAY( STRUCT()[,STRUCT()] - explodes and array and struct into a table +PREHOOK: query: explain vectorization expression +SELECT inline( + ARRAY( + STRUCT (1,'dude!'), + STRUCT (2,'Wheres'), + STRUCT (3,'my car?') + ) +) as (id, text) FROM SRC limit 2 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization expression +SELECT inline( + ARRAY( + STRUCT (1,'dude!'), + STRUCT (2,'Wheres'), + STRUCT (3,'my car?') + ) +) as (id, text) FROM SRC limit 2 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: array(const struct(1,'dude!'),const struct(2,'Wheres'),const struct(3,'my car?')) (type: array>) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 32000 Basic stats: COMPLETE Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 500 Data size: 32000 Basic stats: COMPLETE Column stats: COMPLETE + function name: inline + Select Operator + expressions: col1 (type: int), col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: SELECT operator: Unable to vectorize custom UDF. Complex type constants not supported: Const struct [1, dude!] + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: 2 + Processor Tree: + ListSink + +PREHOOK: query: SELECT inline( + ARRAY( + STRUCT (1,'dude!'), + STRUCT (2,'Wheres'), + STRUCT (3,'my car?') + ) +) as (id, text) FROM SRC limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT inline( + ARRAY( + STRUCT (1,'dude!'), + STRUCT (2,'Wheres'), + STRUCT (3,'my car?') + ) +) as (id, text) FROM SRC limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +1 dude! +2 Wheres +PREHOOK: query: select * from (SELECT + ARRAY( + STRUCT (1,'dude!'), + STRUCT (2,'Wheres'), + STRUCT (3,'my car?') + ) as value FROM SRC limit 1) input + LATERAL VIEW inline(value) myTable AS id, text +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from (SELECT + ARRAY( + STRUCT (1,'dude!'), + STRUCT (2,'Wheres'), + STRUCT (3,'my car?') + ) as value FROM SRC limit 1) input + LATERAL VIEW inline(value) myTable AS id, text +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +[{"col1":1,"col2":"dude!"},{"col1":2,"col2":"Wheres"},{"col1":3,"col2":"my car?"}] 1 dude! +[{"col1":1,"col2":"dude!"},{"col1":2,"col2":"Wheres"},{"col1":3,"col2":"my car?"}] 2 Wheres +[{"col1":1,"col2":"dude!"},{"col1":2,"col2":"Wheres"},{"col1":3,"col2":"my car?"}] 3 my car?