diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java index 9caa4ed169ba92dbd863e4a2dc6d06ab226a4465..9199127735533f9a324c5ef456786dda10766c46 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java @@ -310,6 +310,10 @@ private Writable createPrimitive(final Object obj, final PrimitiveObjectInspecto } private Writable createObject(final Object obj, final ObjectInspector inspector) throws SerDeException { + if (obj == null) { + return null; + } + switch (inspector.getCategory()) { case STRUCT: return createStruct(obj, (StructObjectInspector) inspector); diff --git a/ql/src/test/queries/clientpositive/parquet_write_correct_definition_levels.q b/ql/src/test/queries/clientpositive/parquet_write_correct_definition_levels.q new file mode 100644 index 0000000000000000000000000000000000000000..172fda63ce7c7f3e5cdb673b1c55bf2d5fea99a1 --- /dev/null +++ b/ql/src/test/queries/clientpositive/parquet_write_correct_definition_levels.q @@ -0,0 +1,18 @@ +CREATE TABLE text_tbl (a STRUCT>) +STORED AS TEXTFILE; + +-- This inserts one NULL row +INSERT OVERWRITE TABLE text_tbl +SELECT IF(false, named_struct("b", named_struct("c", 1)), NULL) +FROM src LIMIT 1; + +-- We test that parquet is written with a level 0 definition +CREATE TABLE parq_tbl +STORED AS PARQUET +AS SELECT * FROM text_tbl; + +SELECT * FROM text_tbl; +SELECT * FROM parq_tbl; + +DROP TABLE text_tbl; +DROP TABLE parq_tbl; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/parquet_write_correct_definition_levels.q.out b/ql/src/test/results/clientpositive/parquet_write_correct_definition_levels.q.out new file mode 100644 index 0000000000000000000000000000000000000000..3aa104080991000d1df227f80f5a60e2d73e9bbf --- /dev/null +++ b/ql/src/test/results/clientpositive/parquet_write_correct_definition_levels.q.out @@ -0,0 +1,75 @@ +PREHOOK: query: CREATE TABLE text_tbl (a STRUCT>) +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@text_tbl +POSTHOOK: query: CREATE TABLE text_tbl (a STRUCT>) +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@text_tbl +PREHOOK: query: -- This inserts one NULL row +INSERT OVERWRITE TABLE text_tbl +SELECT IF(false, named_struct("b", named_struct("c", 1)), NULL) +FROM src LIMIT 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@text_tbl +POSTHOOK: query: -- This inserts one NULL row +INSERT OVERWRITE TABLE text_tbl +SELECT IF(false, named_struct("b", named_struct("c", 1)), NULL) +FROM src LIMIT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@text_tbl +POSTHOOK: Lineage: text_tbl.a EXPRESSION [] +PREHOOK: query: -- We test that parquet is written with a level 0 definition +CREATE TABLE parq_tbl +STORED AS PARQUET +AS SELECT * FROM text_tbl +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@text_tbl +PREHOOK: Output: database:default +PREHOOK: Output: default@parq_tbl +POSTHOOK: query: -- We test that parquet is written with a level 0 definition +CREATE TABLE parq_tbl +STORED AS PARQUET +AS SELECT * FROM text_tbl +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@text_tbl +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parq_tbl +PREHOOK: query: SELECT * FROM text_tbl +PREHOOK: type: QUERY +PREHOOK: Input: default@text_tbl +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM text_tbl +POSTHOOK: type: QUERY +POSTHOOK: Input: default@text_tbl +#### A masked pattern was here #### +NULL +PREHOOK: query: SELECT * FROM parq_tbl +PREHOOK: type: QUERY +PREHOOK: Input: default@parq_tbl +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parq_tbl +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parq_tbl +#### A masked pattern was here #### +NULL +PREHOOK: query: DROP TABLE text_tbl +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@text_tbl +PREHOOK: Output: default@text_tbl +POSTHOOK: query: DROP TABLE text_tbl +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@text_tbl +POSTHOOK: Output: default@text_tbl +PREHOOK: query: DROP TABLE parq_tbl +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@parq_tbl +PREHOOK: Output: default@parq_tbl +POSTHOOK: query: DROP TABLE parq_tbl +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@parq_tbl +POSTHOOK: Output: default@parq_tbl