diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 1a34659..64ea0ec 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -309,6 +309,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ vector_left_outer_join.q,\ vector_left_outer_join2.q,\ vector_leftsemi_mapjoin.q,\ + vector_list_constant.q,\ vector_map_order.q,\ vector_mr_diff_schema_alias.q,\ vector_multi_insert.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index abbbe9a..ca37e36 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -1352,6 +1352,9 @@ private VectorExpression getConstantVectorExpression(Object constantValue, TypeI outCol = ocm.allocateOutputColumn(typeInfo); } if (constantValue == null) { + if (typeInfo.getCategory() != Category.PRIMITIVE) { + throw new HiveException("Complex type constants (" + typeInfo.getCategory() + ") not supported for type name " + typeName); + } return new ConstantVectorExpression(outCol, typeInfo, true); } diff --git ql/src/test/queries/clientpositive/vector_list_constant.q ql/src/test/queries/clientpositive/vector_list_constant.q new file mode 100644 index 0000000..371a0d1 --- /dev/null +++ ql/src/test/queries/clientpositive/vector_list_constant.q @@ -0,0 +1,27 @@ +SET hive.vectorized.execution.enabled=true; +set hive.fetch.task.conversion=none; + +create table testdatatypetable + (c1 int, c2 boolean, c3 double, c4 string, + c5 array, c6 map, c7 map, + c8 struct, + c9 tinyint, c10 smallint, c11 float, c12 bigint, + c13 array>, + c14 map>, + c15 struct>, + c16 array,n:int>>, + c17 timestamp, + c18 decimal(16,7), + c19 binary, + c20 date, + c21 varchar(20), + c22 char(15), + c23 binary + ) partitioned by (dt STRING); + +load data local inpath '../../data/files/datatypes.txt' into table testdatatypetable PARTITION (dt='20090619'); + +explain vectorization detail +select +c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, c1*2, +sentences(null, null, null) as b, c17, c18, c20, c21, c22, c23 from testdatatypetable limit 1; \ No newline at end of file diff --git ql/src/test/results/clientpositive/llap/vector_list_constant.q.out ql/src/test/results/clientpositive/llap/vector_list_constant.q.out new file mode 100644 index 0000000..08f3099 --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_list_constant.q.out @@ -0,0 +1,106 @@ +PREHOOK: query: create table testdatatypetable + (c1 int, c2 boolean, c3 double, c4 string, + c5 array, c6 map, c7 map, + c8 struct, + c9 tinyint, c10 smallint, c11 float, c12 bigint, + c13 array>, + c14 map>, + c15 struct>, + c16 array,n:int>>, + c17 timestamp, + c18 decimal(16,7), + c19 binary, + c20 date, + c21 varchar(20), + c22 char(15), + c23 binary + ) partitioned by (dt STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testdatatypetable +POSTHOOK: query: create table testdatatypetable + (c1 int, c2 boolean, c3 double, c4 string, + c5 array, c6 map, c7 map, + c8 struct, + c9 tinyint, c10 smallint, c11 float, c12 bigint, + c13 array>, + c14 map>, + c15 struct>, + c16 array,n:int>>, + c17 timestamp, + c18 decimal(16,7), + c19 binary, + c20 date, + c21 varchar(20), + c22 char(15), + c23 binary + ) partitioned by (dt STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testdatatypetable +PREHOOK: query: load data local inpath '../../data/files/datatypes.txt' into table testdatatypetable PARTITION (dt='20090619') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testdatatypetable +POSTHOOK: query: load data local inpath '../../data/files/datatypes.txt' into table testdatatypetable PARTITION (dt='20090619') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testdatatypetable +POSTHOOK: Output: default@testdatatypetable@dt=20090619 +PREHOOK: query: explain vectorization detail +select +c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, c1*2, +sentences(null, null, null) as b, c17, c18, c20, c21, c22, c23 from testdatatypetable limit 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select +c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, c1*2, +sentences(null, null, null) as b, c17, c18, c20, c21, c22, c23 from testdatatypetable limit 1 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: testdatatypetable + Statistics: Num rows: 1 Data size: 6057 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: c1 (type: int), c2 (type: boolean), c3 (type: double), c4 (type: string), c5 (type: array), c6 (type: map), c7 (type: map), c8 (type: struct), c9 (type: tinyint), c10 (type: smallint), c11 (type: float), c12 (type: bigint), (c1 * 2) (type: int), Const array> null (type: array>), c17 (type: timestamp), c18 (type: decimal(16,7)), c20 (type: date), c21 (type: varchar(20)), c22 (type: char(15)), c23 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 + Statistics: Num rows: 1 Data size: 6057 Basic stats: PARTIAL Column stats: NONE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 6057 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 6057 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: SELECT operator: Complex type constants (LIST) not supported for type name array> + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/vector_list_constant.q.out ql/src/test/results/clientpositive/vector_list_constant.q.out new file mode 100644 index 0000000..f93496c --- /dev/null +++ ql/src/test/results/clientpositive/vector_list_constant.q.out @@ -0,0 +1,101 @@ +PREHOOK: query: create table testdatatypetable + (c1 int, c2 boolean, c3 double, c4 string, + c5 array, c6 map, c7 map, + c8 struct, + c9 tinyint, c10 smallint, c11 float, c12 bigint, + c13 array>, + c14 map>, + c15 struct>, + c16 array,n:int>>, + c17 timestamp, + c18 decimal(16,7), + c19 binary, + c20 date, + c21 varchar(20), + c22 char(15), + c23 binary + ) partitioned by (dt STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testdatatypetable +POSTHOOK: query: create table testdatatypetable + (c1 int, c2 boolean, c3 double, c4 string, + c5 array, c6 map, c7 map, + c8 struct, + c9 tinyint, c10 smallint, c11 float, c12 bigint, + c13 array>, + c14 map>, + c15 struct>, + c16 array,n:int>>, + c17 timestamp, + c18 decimal(16,7), + c19 binary, + c20 date, + c21 varchar(20), + c22 char(15), + c23 binary + ) partitioned by (dt STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testdatatypetable +PREHOOK: query: load data local inpath '../../data/files/datatypes.txt' into table testdatatypetable PARTITION (dt='20090619') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testdatatypetable +POSTHOOK: query: load data local inpath '../../data/files/datatypes.txt' into table testdatatypetable PARTITION (dt='20090619') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testdatatypetable +POSTHOOK: Output: default@testdatatypetable@dt=20090619 +PREHOOK: query: explain vectorization detail +select +c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, c1*2, +sentences(null, null, null) as b, c17, c18, c20, c21, c22, c23 from testdatatypetable limit 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select +c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, c1*2, +sentences(null, null, null) as b, c17, c18, c20, c21, c22, c23 from testdatatypetable limit 1 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: testdatatypetable + Statistics: Num rows: 1 Data size: 3250 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: c1 (type: int), c2 (type: boolean), c3 (type: double), c4 (type: string), c5 (type: array), c6 (type: map), c7 (type: map), c8 (type: struct), c9 (type: tinyint), c10 (type: smallint), c11 (type: float), c12 (type: bigint), (c1 * 2) (type: int), Const array> null (type: array>), c17 (type: timestamp), c18 (type: decimal(16,7)), c20 (type: date), c21 (type: varchar(20)), c22 (type: char(15)), c23 (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19 + Statistics: Num rows: 1 Data size: 3250 Basic stats: PARTIAL Column stats: NONE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 3250 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 3250 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + notVectorizedReason: SELECT operator: Complex type constants (LIST) not supported for type name array> + vectorized: false + + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink +