diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index aeb6211..faa4279 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -760,6 +760,7 @@ minillaplocal.query.files=\ vector_orc_null_check.q,\ vector_order_null.q,\ vector_outer_reference_windowed.q,\ + vector_parquet_nested_two_level_complex.q,\ vector_partitioned_date_time.q,\ vector_ptf_1.q,\ vector_ptf_part_simple.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index f4e8207..7afbf04 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -221,6 +221,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -1288,6 +1290,7 @@ private void addVectorPartitionDesc(PartitionDesc pd, VectorPartitionDesc vpd, */ private boolean verifyAndSetVectorPartDesc( PartitionDesc pd, boolean isFullAcidTable, + List allTypeInfoList, Set inputFileFormatClassNameSet, Map vectorPartitionDescMap, Set enabledConditionsMetSet, ArrayList enabledConditionsNotMetList, @@ -1332,8 +1335,13 @@ private boolean verifyAndSetVectorPartDesc( if (useVectorizedInputFileFormat) { - if (isInputFileFormatVectorized && !isInputFormatExcluded(inputFileFormatClassName, - vectorizedInputFormatExcludes)) { + if (isInputFileFormatVectorized && + !isInputFormatExcluded( + inputFileFormatClassName, + vectorizedInputFormatExcludes) && + !hasUnsupportedVectorizedParquetDataType( + inputFileFormatClass, + allTypeInfoList)) { addVectorizedInputFileFormatSupport( newSupportSet, isInputFileFormatVectorized, inputFileFormatClass); @@ -1517,6 +1525,57 @@ private boolean isInputFormatExcluded(String inputFileFormatClassName, Collectio return false; } + private boolean hasUnsupportedVectorizedParquetDataType( + Class inputFileFormatClass, List allTypeInfoList) { + if (!inputFileFormatClass.equals(org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat.class)) { + return false; + } + + /* + * Currently, VectorizedParquetRecordReader cannot handle nested complex types. + */ + for (TypeInfo typeInfo : allTypeInfoList) { + if (!(typeInfo instanceof PrimitiveTypeInfo)) { + switch (typeInfo.getCategory()) { + case LIST: + if (!(((ListTypeInfo) typeInfo).getListElementTypeInfo() instanceof PrimitiveTypeInfo)) { + return true; + } + break; + case MAP: + { + MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; + if (!(mapTypeInfo.getMapKeyTypeInfo() instanceof PrimitiveTypeInfo)) { + return true; + } + if (!(mapTypeInfo.getMapValueTypeInfo() instanceof PrimitiveTypeInfo)) { + return true; + } + } + break; + case STRUCT: + { + StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; + List fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); + for (TypeInfo fieldTypeInfo : fieldTypeInfos) { + if (!(fieldTypeInfo instanceof PrimitiveTypeInfo)) { + return true; + } + } + } + break; + case UNION: + // Not supported at all. + return false; + default: + throw new RuntimeException( + "Unsupported complex type category " + typeInfo.getCategory()); + } + } + } + return false; + } + private void setValidateInputFormatAndSchemaEvolutionExplain(MapWork mapWork, Set inputFileFormatClassNameSet, Map vectorPartitionDescMap, @@ -1594,6 +1653,7 @@ private void setValidateInputFormatAndSchemaEvolutionExplain(MapWork mapWork, final boolean isVerifiedVectorPartDesc = verifyAndSetVectorPartDesc( partDesc, isFullAcidTable, + allTypeInfoList, inputFileFormatClassNameSet, vectorPartitionDescMap, enabledConditionsMetSet, enabledConditionsNotMetList, diff --git ql/src/test/queries/clientpositive/vector_parquet_nested_two_level_complex.q ql/src/test/queries/clientpositive/vector_parquet_nested_two_level_complex.q new file mode 100644 index 0000000..70480e6 --- /dev/null +++ ql/src/test/queries/clientpositive/vector_parquet_nested_two_level_complex.q @@ -0,0 +1,67 @@ +set hive.vectorized.execution.enabled=true; +set hive.explain.user=false; +set hive.fetch.task.conversion=none; + +create table nested_array_array_table ( +nested_array_array array>) +STORED AS PARQUET; + +create table nested_array_map_table ( +nested_array_map array>) +STORED AS PARQUET; + +create table nested_array_struct_table ( +nested_array_map array>) +STORED AS PARQUET; + +create table nested_map_array_table ( +nested_map_array map>) +STORED AS PARQUET; + +create table nested_map_map_table ( +nested_map_map map>) +STORED AS PARQUET; + +create table nested_map_struct_table ( +nested_map_struct map>) +STORED AS PARQUET; + +create table nested_struct_array_table ( +nested_struct_array struct>) +STORED AS PARQUET; + +create table nested_struct_map_table ( +nested_struct_map struct>) +STORED AS PARQUET; + +create table nested_struct_struct_table ( +nested_struct_struct struct>) +STORED AS PARQUET; + + +EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_array_array_table; + +EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_array_map_table; + +EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_array_map_table; + +EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_map_array_table; + +EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_map_map_table; + +EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_map_struct_table; + +EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_struct_array_table; + +EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_struct_map_table; + +EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_struct_struct_table; diff --git ql/src/test/results/clientpositive/llap/vector_parquet_nested_two_level_complex.q.out ql/src/test/results/clientpositive/llap/vector_parquet_nested_two_level_complex.q.out new file mode 100644 index 0000000..dc791fe --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_parquet_nested_two_level_complex.q.out @@ -0,0 +1,603 @@ +PREHOOK: query: create table nested_array_array_table ( +nested_array_array array>) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nested_array_array_table +POSTHOOK: query: create table nested_array_array_table ( +nested_array_array array>) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nested_array_array_table +PREHOOK: query: create table nested_array_map_table ( +nested_array_map array>) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nested_array_map_table +POSTHOOK: query: create table nested_array_map_table ( +nested_array_map array>) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nested_array_map_table +PREHOOK: query: create table nested_array_struct_table ( +nested_array_map array>) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nested_array_struct_table +POSTHOOK: query: create table nested_array_struct_table ( +nested_array_map array>) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nested_array_struct_table +PREHOOK: query: create table nested_map_array_table ( +nested_map_array map>) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nested_map_array_table +POSTHOOK: query: create table nested_map_array_table ( +nested_map_array map>) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nested_map_array_table +PREHOOK: query: create table nested_map_map_table ( +nested_map_map map>) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nested_map_map_table +POSTHOOK: query: create table nested_map_map_table ( +nested_map_map map>) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nested_map_map_table +PREHOOK: query: create table nested_map_struct_table ( +nested_map_struct map>) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nested_map_struct_table +POSTHOOK: query: create table nested_map_struct_table ( +nested_map_struct map>) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nested_map_struct_table +PREHOOK: query: create table nested_struct_array_table ( +nested_struct_array struct>) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nested_struct_array_table +POSTHOOK: query: create table nested_struct_array_table ( +nested_struct_array struct>) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nested_struct_array_table +PREHOOK: query: create table nested_struct_map_table ( +nested_struct_map struct>) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nested_struct_map_table +POSTHOOK: query: create table nested_struct_map_table ( +nested_struct_map struct>) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nested_struct_map_table +PREHOOK: query: create table nested_struct_struct_table ( +nested_struct_struct struct>) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nested_struct_struct_table +POSTHOOK: query: create table nested_struct_struct_table ( +nested_struct_struct struct>) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nested_struct_struct_table +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_array_array_table +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_array_array_table +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_array_array_table + Statistics: Num rows: 1 Data size: 1280 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:nested_array_array:array>, 1:ROW__ID:struct] + Select Operator + expressions: nested_array_array (type: array>) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1 Data size: 1280 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 1280 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [0] + dataColumns: nested_array_array:array> + partitionColumnCount: 0 + scratchColumnTypeNames: [] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_array_map_table +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_array_map_table +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_array_map_table + Statistics: Num rows: 1 Data size: 9280 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:nested_array_map:array>, 1:ROW__ID:struct] + Select Operator + expressions: nested_array_map (type: array>) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1 Data size: 9280 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 9280 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [0] + dataColumns: nested_array_map:array> + partitionColumnCount: 0 + scratchColumnTypeNames: [] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_array_map_table +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_array_map_table +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_array_map_table + Statistics: Num rows: 1 Data size: 9280 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:nested_array_map:array>, 1:ROW__ID:struct] + Select Operator + expressions: nested_array_map (type: array>) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1 Data size: 9280 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 9280 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [0] + dataColumns: nested_array_map:array> + partitionColumnCount: 0 + scratchColumnTypeNames: [] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_map_array_table +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_map_array_table +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_map_array_table + Statistics: Num rows: 1 Data size: 856 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: nested_map_array (type: map>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 856 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 856 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: false + enabledConditionsNotMet: Row deserialization of vectorized input format not supported IS false, hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_map_map_table +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_map_map_table +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_map_map_table + Statistics: Num rows: 1 Data size: 1656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: nested_map_map (type: map>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: false + enabledConditionsNotMet: Row deserialization of vectorized input format not supported IS false, hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_map_struct_table +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_map_struct_table +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_map_struct_table + Statistics: Num rows: 1 Data size: 800 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: nested_map_struct (type: map>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 800 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 800 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: false + enabledConditionsNotMet: Row deserialization of vectorized input format not supported IS false, hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_struct_array_table +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_struct_array_table +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_struct_array_table + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: nested_struct_array (type: struct>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: false + enabledConditionsNotMet: Row deserialization of vectorized input format not supported IS false, hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_struct_map_table +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_struct_map_table +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_struct_map_table + Statistics: Num rows: 1 Data size: 1168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: nested_struct_map (type: struct>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 1168 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1168 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: false + enabledConditionsNotMet: Row deserialization of vectorized input format not supported IS false, hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_struct_struct_table +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT * FROM nested_struct_struct_table +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: nested_struct_struct_table + Statistics: Num rows: 1 Data size: 312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: nested_struct_struct (type: struct>) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: false + enabledConditionsNotMet: Row deserialization of vectorized input format not supported IS false, hive.vectorized.use.vectorized.input.format IS true AND hive.vectorized.input.format.excludes NOT CONTAINS org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat IS false + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink +