diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index d2e077b..28c14eb 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -370,6 +370,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ vector_left_outer_join.q,\ vector_left_outer_join2.q,\ vector_leftsemi_mapjoin.q,\ + vector_map_order.q,\ vector_mr_diff_schema_alias.q,\ vector_multi_insert.q,\ vector_null_projection.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java index d0961b3..152d75b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hive.ql.exec.vector; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -397,7 +397,7 @@ public Object extractRowColumn( final int offset = (int) mapColumnVector.offsets[adjustedIndex]; final int size = (int) mapColumnVector.lengths[adjustedIndex]; - final Map map = new HashMap(); + final Map map = new LinkedHashMap(); for (int i = 0; i < size; i++) { final Object key = extractRowColumn( mapColumnVector.keys, diff --git ql/src/test/queries/clientpositive/vector_map_order.q ql/src/test/queries/clientpositive/vector_map_order.q new file mode 100644 index 0000000..b1f05d5 --- /dev/null +++ ql/src/test/queries/clientpositive/vector_map_order.q @@ -0,0 +1,15 @@ +SET hive.vectorized.execution.enabled=true; +set hive.fetch.task.conversion=none; + +create table map_table (foo STRING , bar MAP) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY '\t' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +STORED AS TEXTFILE; + +load data local inpath "../../data/files/map_table.txt" overwrite into table map_table; + +explain vectorization detail +select * from map_table; +select * from map_table; \ No newline at end of file diff --git ql/src/test/results/clientpositive/llap/vector_map_order.q.out ql/src/test/results/clientpositive/llap/vector_map_order.q.out new file mode 100644 index 0000000..88748fa --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_map_order.q.out @@ -0,0 +1,106 @@ +PREHOOK: query: create table map_table (foo STRING , bar MAP) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY '\t' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@map_table +POSTHOOK: query: create table map_table (foo STRING , bar MAP) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY '\t' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@map_table +PREHOOK: query: load data local inpath "../../data/files/map_table.txt" overwrite into table map_table +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@map_table +POSTHOOK: query: load data local inpath "../../data/files/map_table.txt" overwrite into table map_table +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@map_table +PREHOOK: query: explain vectorization detail +select * from map_table +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select * from map_table +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: map_table + Statistics: Num rows: 1 Data size: 1104 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:foo:string, 1:bar:map, 2:ROW__ID:struct] + Select Operator + expressions: foo (type: string), bar (type: map) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 1 Data size: 1104 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 1104 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + vectorizationSupportRemovedReasons: [DECIMAL_64 disabled because LLAP is enabled] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + includeColumns: [0, 1] + dataColumns: foo:string, bar:map + partitionColumnCount: 0 + scratchColumnTypeNames: [] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from map_table +PREHOOK: type: QUERY +PREHOOK: Input: default@map_table +#### A masked pattern was here #### +POSTHOOK: query: select * from map_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@map_table +#### A masked pattern was here #### +foo1 {"k1":"v1","k2":"v2","k3":"v3"} +foo2 {"k21":"v21","k22":"v22","k31":"v31"} diff --git ql/src/test/results/clientpositive/vector_map_order.q.out ql/src/test/results/clientpositive/vector_map_order.q.out new file mode 100644 index 0000000..6fe6943 --- /dev/null +++ ql/src/test/results/clientpositive/vector_map_order.q.out @@ -0,0 +1,101 @@ +PREHOOK: query: create table map_table (foo STRING , bar MAP) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY '\t' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@map_table +POSTHOOK: query: create table map_table (foo STRING , bar MAP) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY '\t' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@map_table +PREHOOK: query: load data local inpath "../../data/files/map_table.txt" overwrite into table map_table +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@map_table +POSTHOOK: query: load data local inpath "../../data/files/map_table.txt" overwrite into table map_table +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@map_table +PREHOOK: query: explain vectorization detail +select * from map_table +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select * from map_table +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: map_table + Statistics: Num rows: 1 Data size: 520 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:foo:string, 1:bar:map, 2:ROW__ID:struct] + Select Operator + expressions: foo (type: string), bar (type: map) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 1 Data size: 520 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 520 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + includeColumns: [0, 1] + dataColumns: foo:string, bar:map + partitionColumnCount: 0 + scratchColumnTypeNames: [] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from map_table +PREHOOK: type: QUERY +PREHOOK: Input: default@map_table +#### A masked pattern was here #### +POSTHOOK: query: select * from map_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@map_table +#### A masked pattern was here #### +foo1 {"k1":"v1","k2":"v2","k3":"v3"} +foo2 {"k21":"v21","k22":"v22","k31":"v31"}