diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 5985dcf..30e0bca 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -748,6 +748,7 @@ minillaplocal.query.files=\ vector_groupby_grouping_window.q,\ vector_groupby_rollup1.q,\ vector_if_expr_2.q,\ + vector_input_testxpath3.q,\ vector_join30.q,\ vector_join_filters.q,\ vector_leftsemi_mapjoin.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index 93212ce..72bf074 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -852,6 +852,20 @@ public VectorExpression getVectorExpression(ExprNodeDesc exprDesc, VectorExpress private VectorExpression getGenericUDFStructField(ExprNodeFieldDesc exprNodeFieldDesc, VectorExpressionDescriptor.Mode mode, TypeInfo returnType) throws HiveException { + ExprNodeDesc parentNodeDesc = exprNodeFieldDesc.getDesc(); + TypeInfo parentTypeInfo = parentNodeDesc.getTypeInfo(); + if (parentTypeInfo.getCategory() != Category.STRUCT) { + + // We don't support the forms like get field a from type array> that + // returns a type array. Unfortunately, we do not have a real GenericUDF that + // we can fall back to. GenericUDFStructField is fake. + throw new HiveException( + "Could not create native vectorize expression for:" + exprNodeFieldDesc.getExprString() + + " (type: " + exprNodeFieldDesc.getTypeString() + ")" + + " -- expecting a field reference from a STRUCT but found " + + parentNodeDesc.getExprString() + " is type " + parentNodeDesc.getTypeString() + ". " + + "The alternative of using the VectorUDFAdaptor is not available"); + } // set the arguments for GenericUDFStructField List children = new ArrayList<>(2); children.add(exprNodeFieldDesc.getDesc()); diff --git ql/src/test/queries/clientpositive/input_testxpath3.q ql/src/test/queries/clientpositive/input_testxpath3.q index 1b57ead..fb7e609 100644 --- ql/src/test/queries/clientpositive/input_testxpath3.q +++ ql/src/test/queries/clientpositive/input_testxpath3.q @@ -1,3 +1,5 @@ +SET hive.vectorized.execution.enabled=false; + EXPLAIN FROM src_thrift SELECT src_thrift.mstringstring['key_9'], src_thrift.lintstring.myint; diff --git ql/src/test/queries/clientpositive/vector_input_testxpath3.q ql/src/test/queries/clientpositive/vector_input_testxpath3.q new file mode 100644 index 0000000..fac17cf --- /dev/null +++ ql/src/test/queries/clientpositive/vector_input_testxpath3.q @@ -0,0 +1,9 @@ +SET hive.vectorized.execution.enabled=true; +set hive.fetch.task.conversion=none; + +EXPLAIN VECTORIZATION EXPRESSION +FROM src_thrift +SELECT src_thrift.mstringstring['key_9'], src_thrift.lintstring.myint; + +FROM src_thrift +SELECT src_thrift.mstringstring['key_9'], src_thrift.lintstring.myint; diff --git ql/src/test/results/clientpositive/llap/vector_input_testxpath3.q.out ql/src/test/results/clientpositive/llap/vector_input_testxpath3.q.out new file mode 100644 index 0000000..843993f --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_input_testxpath3.q.out @@ -0,0 +1,73 @@ +PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +FROM src_thrift +SELECT src_thrift.mstringstring['key_9'], src_thrift.lintstring.myint +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION +FROM src_thrift +SELECT src_thrift.mstringstring['key_9'], src_thrift.lintstring.myint +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_thrift + Statistics: Num rows: 11 Data size: 38280 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: mstringstring['key_9'] (type: string), lintstring.myint (type: array) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 38280 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 11 Data size: 38280 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat + notVectorizedReason: SELECT operator: Could not create native vectorize expression for:lintstring.myint (type: array) -- expecting a field reference from a STRUCT but found lintstring is type array>. The alternative of using the VectorUDFAdaptor is not available + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: FROM src_thrift +SELECT src_thrift.mstringstring['key_9'], src_thrift.lintstring.myint +PREHOOK: type: QUERY +PREHOOK: Input: default@src_thrift +#### A masked pattern was here #### +POSTHOOK: query: FROM src_thrift +SELECT src_thrift.mstringstring['key_9'], src_thrift.lintstring.myint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_thrift +#### A masked pattern was here #### +NULL [0] +NULL [1] +NULL [4] +NULL [9] +NULL [16] +NULL [25] +NULL [36] +NULL [49] +NULL [64] +value_9 [81] +NULL NULL