diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 3d8bae8609..e0805a74c3 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -43,6 +43,7 @@ minitez.query.files=acid_vectorization_original_tez.q,\ hybridgrace_hashjoin_1.q,\ hybridgrace_hashjoin_2.q,\ multi_count_distinct.q,\ + nested_complex2.q,\ tez-tag.q,\ tez_union_with_udf.q @@ -54,6 +55,7 @@ minillap.shared.query.files=insert_into1.q,\ mapreduce2.q,\ mm_all.q,\ mm_cttas.q,\ + nested_complex2.q,\ orc_merge1.q,\ orc_merge10.q,\ orc_merge2.q,\ diff --git ql/src/test/queries/clientpositive/nested_complex2.q ql/src/test/queries/clientpositive/nested_complex2.q new file mode 100644 index 0000000000..f9244d5dbe --- /dev/null +++ ql/src/test/queries/clientpositive/nested_complex2.q @@ -0,0 +1,25 @@ +set hive.vectorized.execution.enabled=true; +set hive.mapred.mode=nonstrict; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +-- Array nested in map + +CREATE TABLE srctable(a map>) STORED AS TEXTFILE; +create table desttable(c1 map>) stored as orc; +insert into srctable values (map(1, array(1, 2, 3))); +select a from srctable; + +insert into desttable select a from srctable; +select * from desttable; + + +-- Struct nested in map + +create table structinmapsrc(a map>) STORED AS TEXTFILE; +create table structinmapdest(a map>) stored as orc; +insert into structinmapsrc select map(1, named_struct('s', 'sss', 'i', 2)); +select * from structinmapsrc; + +insert into structinmapdest select a from structinmapsrc; +select * from structinmapdest; diff --git ql/src/test/results/clientpositive/tez/nested_complex2.q.out ql/src/test/results/clientpositive/tez/nested_complex2.q.out new file mode 100644 index 0000000000..eeb0c25bd2 --- /dev/null +++ ql/src/test/results/clientpositive/tez/nested_complex2.q.out @@ -0,0 +1,104 @@ +PREHOOK: query: CREATE TABLE srctable(a map>) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srctable +POSTHOOK: query: CREATE TABLE srctable(a map>) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srctable +PREHOOK: query: create table desttable(c1 map>) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@desttable +POSTHOOK: query: create table desttable(c1 map>) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@desttable +PREHOOK: query: insert into srctable values (map(1, array(1, 2, 3))) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@srctable +POSTHOOK: query: insert into srctable values (map(1, array(1, 2, 3))) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@srctable +POSTHOOK: Lineage: srctable.a SCRIPT [] +PREHOOK: query: select a from srctable +PREHOOK: type: QUERY +PREHOOK: Input: default@srctable +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select a from srctable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srctable +POSTHOOK: Output: hdfs://### HDFS PATH ### +{1:[1,2,3]} +PREHOOK: query: insert into desttable select a from srctable +PREHOOK: type: QUERY +PREHOOK: Input: default@srctable +PREHOOK: Output: default@desttable +POSTHOOK: query: insert into desttable select a from srctable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srctable +POSTHOOK: Output: default@desttable +POSTHOOK: Lineage: desttable.c1 SIMPLE [(srctable)srctable.FieldSchema(name:a, type:map>, comment:null), ] +PREHOOK: query: select * from desttable +PREHOOK: type: QUERY +PREHOOK: Input: default@desttable +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from desttable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@desttable +POSTHOOK: Output: hdfs://### HDFS PATH ### +{1:[1,2,3]} +PREHOOK: query: create table structinmapsrc(a map>) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@structinmapsrc +POSTHOOK: query: create table structinmapsrc(a map>) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@structinmapsrc +PREHOOK: query: create table structinmapdest(a map>) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@structinmapdest +POSTHOOK: query: create table structinmapdest(a map>) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@structinmapdest +PREHOOK: query: insert into structinmapsrc select map(1, named_struct('s', 'sss', 'i', 2)) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@structinmapsrc +POSTHOOK: query: insert into structinmapsrc select map(1, named_struct('s', 'sss', 'i', 2)) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@structinmapsrc +POSTHOOK: Lineage: structinmapsrc.a EXPRESSION [] +PREHOOK: query: select * from structinmapsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@structinmapsrc +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from structinmapsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@structinmapsrc +POSTHOOK: Output: hdfs://### HDFS PATH ### +{1:{"s":"sss","i":2}} +PREHOOK: query: insert into structinmapdest select a from structinmapsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@structinmapsrc +PREHOOK: Output: default@structinmapdest +POSTHOOK: query: insert into structinmapdest select a from structinmapsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@structinmapsrc +POSTHOOK: Output: default@structinmapdest +POSTHOOK: Lineage: structinmapdest.a SIMPLE [(structinmapsrc)structinmapsrc.FieldSchema(name:a, type:map>, comment:null), ] +PREHOOK: query: select * from structinmapdest +PREHOOK: type: QUERY +PREHOOK: Input: default@structinmapdest +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from structinmapdest +POSTHOOK: type: QUERY +POSTHOOK: Input: default@structinmapdest +POSTHOOK: Output: hdfs://### HDFS PATH ### +{1:{"s":"sss","i":2}} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java index 7b8aae4a92..ff255383d5 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java @@ -1029,13 +1029,21 @@ public boolean readComplexField() throws IOException { // When data is prematurely ended the fieldPosition will be 1 more than the end. Preconditions.checkState(fieldPosition <= complexFieldEnd + 1); + final boolean isParentMap = isParentMap(); + if (isParentMap) { + currentLevel++; + } final int fieldEnd = parseComplexField(fieldPosition, complexFieldEnd, currentLevel); listHelper.fieldPosition = fieldEnd + 1; // Move past separator. currentFieldStart = fieldPosition; currentFieldLength = fieldEnd - fieldPosition; - return doReadField(listHelper.elementField); + boolean result = doReadField(listHelper.elementField); + if (isParentMap) { + currentLevel--; + } + return result; } case MAP: { @@ -1099,6 +1107,11 @@ public boolean readComplexField() throws IOException { final int nextFieldIndex = structHelper.nextFieldIndex; final Field[] fields = structHelper.fields; final int fieldEnd; + final boolean isParentMap = isParentMap(); + if (isParentMap) { + currentLevel++; + } + boolean result; if (nextFieldIndex != fields.length - 1) { // Parse until field separator (currentLevel). @@ -1108,7 +1121,7 @@ public boolean readComplexField() throws IOException { currentFieldLength = fieldEnd - fieldPosition; - return doReadField(fields[structHelper.nextFieldIndex++]); + result = doReadField(fields[structHelper.nextFieldIndex++]); } else { // Parse until field separator (currentLevel). @@ -1116,7 +1129,7 @@ public boolean readComplexField() throws IOException { currentFieldLength = fieldEnd - fieldPosition; structHelper.nextFieldIndex = 0; - boolean result = doReadField(fields[fields.length - 1]); + result = doReadField(fields[fields.length - 1]); if (!isEscaped) { @@ -1127,9 +1140,11 @@ public boolean readComplexField() throws IOException { // We must parse to get the escape count. parseComplexField(fieldPosition, complexFieldEnd, currentLevel - 1); } - - return result; } + if (isParentMap) { + currentLevel--; + } + return result; } case UNION: {