From c974ac9f064bae89beb0528af0b5b029296a1569 Mon Sep 17 00:00:00 2001 From: Naresh P R Date: Mon, 5 Aug 2019 23:53:11 +0530 Subject: [PATCH] HIVE-16587 - NPE when inserting complex types with nested null values --- .../resources/testconfiguration.properties | 3 +- .../hadoop/hive/ql/stats/StatsUtils.java | 4 +- .../tez_complextype_with_null.q | 15 ++++++ .../tez/tez_complextype_with_null.q.out | 50 +++++++++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/tez_complextype_with_null.q create mode 100644 ql/src/test/results/clientpositive/tez/tez_complextype_with_null.q.out diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index f390dc5ddd..84c2042676 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -42,7 +42,8 @@ minitez.query.files=acid_vectorization_original_tez.q,\ multi_count_distinct.q,\ tez-tag.q,\ tez_union_with_udf.q,\ - tez_union_udtf.q + tez_union_udtf.q,\ + tez_complextype_with_null.q minillap.shared.query.files=insert_into1.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index e0f401beb6..f621e5cd77 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -1344,11 +1344,11 @@ public static long getSizeOfStruct(StandardConstantStructObjectInspector soi) { public static long getWritableSize(ObjectInspector oi, Object value) { if (oi instanceof WritableStringObjectInspector) { WritableStringObjectInspector woi = (WritableStringObjectInspector) oi; - return JavaDataModel.get().lengthForStringOfLength( + return value == null ? 0 : JavaDataModel.get().lengthForStringOfLength( woi.getPrimitiveWritableObject(value).getLength()); } else if (oi instanceof WritableBinaryObjectInspector) { WritableBinaryObjectInspector woi = (WritableBinaryObjectInspector) oi; - return JavaDataModel.get().lengthForByteArrayOfSize( + return value == null ? 0 : JavaDataModel.get().lengthForByteArrayOfSize( woi.getPrimitiveWritableObject(value).getLength()); } else if (oi instanceof WritableBooleanObjectInspector) { return JavaDataModel.get().primitive1(); diff --git a/ql/src/test/queries/clientpositive/tez_complextype_with_null.q b/ql/src/test/queries/clientpositive/tez_complextype_with_null.q new file mode 100644 index 0000000000..7a0f240cdb --- /dev/null +++ b/ql/src/test/queries/clientpositive/tez_complextype_with_null.q @@ -0,0 +1,15 @@ +CREATE TABLE complex1 ( + c0 int, + c1 array, + c2 map, + c3 struct>, + c4 array>>); + +INSERT INTO complex1 + SELECT 3, + array(1, 2, null), + map(1, 'one', 2, null), + named_struct('f1', cast(null as int), 'f2', cast(null as string), 'f3', array(1, 2, null)), + array(named_struct('f1', 11, 'f2', 'two', 'f3', array(2, 3, 4))); + +select * from complex1; diff --git a/ql/src/test/results/clientpositive/tez/tez_complextype_with_null.q.out b/ql/src/test/results/clientpositive/tez/tez_complextype_with_null.q.out new file mode 100644 index 0000000000..f20151d746 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/tez_complextype_with_null.q.out @@ -0,0 +1,50 @@ +PREHOOK: query: CREATE TABLE complex1 ( + c0 int, + c1 array, + c2 map, + c3 struct>, + c4 array>>) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@complex1 +POSTHOOK: query: CREATE TABLE complex1 ( + c0 int, + c1 array, + c2 map, + c3 struct>, + c4 array>>) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@complex1 +PREHOOK: query: INSERT INTO complex1 + SELECT 3, + array(1, 2, null), + map(1, 'one', 2, null), + named_struct('f1', cast(null as int), 'f2', cast(null as string), 'f3', array(1, 2, null)), + array(named_struct('f1', 11, 'f2', 'two', 'f3', array(2, 3, 4))) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@complex1 +POSTHOOK: query: INSERT INTO complex1 + SELECT 3, + array(1, 2, null), + map(1, 'one', 2, null), + named_struct('f1', cast(null as int), 'f2', cast(null as string), 'f3', array(1, 2, null)), + array(named_struct('f1', 11, 'f2', 'two', 'f3', array(2, 3, 4))) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@complex1 +POSTHOOK: Lineage: complex1.c0 SIMPLE [] +POSTHOOK: Lineage: complex1.c1 EXPRESSION [] +POSTHOOK: Lineage: complex1.c2 EXPRESSION [] +POSTHOOK: Lineage: complex1.c3 EXPRESSION [] +POSTHOOK: Lineage: complex1.c4 EXPRESSION [] +PREHOOK: query: select * from complex1 +PREHOOK: type: QUERY +PREHOOK: Input: default@complex1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from complex1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@complex1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +3 [1,2,null] {1:"one",2:null} {"f1":null,"f2":null,"f3":[1,2,null]} [{"f1":11,"f2":"two","f3":[2,3,4]}] -- 2.18.0