diff --git data/files/parquet_fake_table.txt data/files/parquet_fake_table.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ data/files/parquet_fake_table.txt @@ -0,0 +1 @@ +1 diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ArrayWritableGroupConverter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ArrayWritableGroupConverter.java index 582a5df..052b36d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ArrayWritableGroupConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ArrayWritableGroupConverter.java @@ -54,6 +54,7 @@ public void start() { if (isMap) { mapPairContainer = new Writable[2]; } + currentValue = null; } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java index b689336..9f2b4aa 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java @@ -166,7 +166,7 @@ private ArrayWritable createStruct(final Object obj, final StructObjectInspector return new ArrayWritable(Writable.class, arr); } - private Writable createMap(final Object obj, final MapObjectInspector inspector) + private ArrayWritable createMap(final Object obj, final MapObjectInspector inspector) throws SerDeException { final Map sourceMap = inspector.getMap(obj); final ObjectInspector keyInspector = inspector.getMapKeyObjectInspector(); @@ -178,16 +178,12 @@ private Writable createMap(final Object obj, final MapObjectInspector inspector) final Writable key = createObject(keyValue.getKey(), keyInspector); final Writable value = createObject(keyValue.getValue(), valueInspector); if (key != null) { - Writable[] arr = new Writable[2]; - arr[0] = key; - arr[1] = value; - array.add(new ArrayWritable(Writable.class, arr)); + array.add(new ArrayWritable(Writable.class, new Writable[] {key, value})); } } } if (array.size() > 0) { - final ArrayWritable subArray = new ArrayWritable(ArrayWritable.class, - array.toArray(new ArrayWritable[array.size()])); + final ArrayWritable subArray = new ArrayWritable(ArrayWritable.class, array.toArray(new ArrayWritable[array.size()])); return new ArrayWritable(Writable.class, new Writable[] {subArray}); } else { return null; @@ -198,18 +194,14 @@ private ArrayWritable createArray(final Object obj, final ListObjectInspector in throws SerDeException { final List sourceArray = inspector.getList(obj); final ObjectInspector subInspector = inspector.getListElementObjectInspector(); - final List array = new ArrayList(); + final List array = new ArrayList(); if (sourceArray != null) { for (final Object curObj : sourceArray) { - final Writable newObj = createObject(curObj, subInspector); - if (newObj != null) { - array.add(newObj); - } + array.add(new ArrayWritable(Writable.class, new Writable[] {createObject(curObj, subInspector)})); } } if (array.size() > 0) { - final ArrayWritable subArray = new ArrayWritable(array.get(0).getClass(), - array.toArray(new Writable[array.size()])); + final ArrayWritable subArray = new ArrayWritable(ArrayWritable.class, array.toArray(new ArrayWritable[array.size()])); return new ArrayWritable(Writable.class, new Writable[] {subArray}); } else { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java index a98f6be..a3e3fcc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java @@ -74,18 +74,17 @@ private void writeData(final ArrayWritable arr, final GroupType type) { if (fieldType.isPrimitive()) { writePrimitive(value); } else { - recordConsumer.startGroup(); if (value instanceof ArrayWritable) { if (fieldType.asGroupType().getRepetition().equals(Type.Repetition.REPEATED)) { writeArray((ArrayWritable) value, fieldType.asGroupType()); } else { + recordConsumer.startGroup(); writeData((ArrayWritable) value, fieldType.asGroupType()); + recordConsumer.endGroup(); } } else if (value != null) { throw new ParquetEncodingException("This should be an ArrayWritable or MapWritable: " + value); } - - recordConsumer.endGroup(); } recordConsumer.endField(fieldName, field); @@ -96,32 +95,43 @@ private void writeArray(final ArrayWritable array, final GroupType type) { if (array == null) { return; } - final Writable[] subValues = array.get(); + final Writable[] values = array.get(); final int fieldCount = type.getFieldCount(); - for (int field = 0; field < fieldCount; ++field) { - final Type subType = type.getType(field); - recordConsumer.startField(subType.getName(), field); - for (int i = 0; i < subValues.length; ++i) { - final Writable subValue = subValues[i]; - if (subValue != null) { - if (subType.isPrimitive()) { - if (subValue instanceof ArrayWritable) { - writePrimitive(((ArrayWritable) subValue).get()[field]);// 0 ? - } else { - writePrimitive(subValue); - } - } else { - if (!(subValue instanceof ArrayWritable)) { - throw new RuntimeException("This should be a ArrayWritable: " + subValue); - } else { - recordConsumer.startGroup(); - writeData((ArrayWritable) subValue, subType.asGroupType()); - recordConsumer.endGroup(); + + for (int i = 0; i < values.length; ++i) { + recordConsumer.startGroup(); + Writable value = values[i]; // ArrayWritable of 1 elem if list, or of 2 elems if map + + if (!(value instanceof ArrayWritable)) { + throw new ParquetEncodingException("This should be an ArrayWritable but was: " + value); + } + + for (int field = 0; field < fieldCount; ++field) { + Type subType = type.getType(field); // either array element, map key or map value + + Writable subValue = ((ArrayWritable) value).get()[field]; // either array element, map key or map value + + if (subValue != null) { + recordConsumer.startField(subType.getName(), field); + if (subType.isPrimitive()) { + writePrimitive(subValue); + } else { + recordConsumer.startGroup(); + if (subValue instanceof ArrayWritable) { + if (subType.asGroupType().getRepetition().equals(Type.Repetition.REPEATED)) { + throw new ParquetEncodingException("REPEATED level can not be found here."); + } else { + writeData((ArrayWritable) subValue, subType.asGroupType()); + } + } else if (subValue != null) { + throw new ParquetEncodingException("This should be an ArrayWritable or MapWritable: " + subValue); + } + recordConsumer.endGroup(); + } + recordConsumer.endField(subType.getName(), field); } - } } - } - recordConsumer.endField(subType.getName(), field); + recordConsumer.endGroup(); } } diff --git ql/src/test/queries/clientpositive/parquet_complex_type_nested.q ql/src/test/queries/clientpositive/parquet_complex_type_nested.q new file mode 100644 index 0000000..1096262 --- /dev/null +++ ql/src/test/queries/clientpositive/parquet_complex_type_nested.q @@ -0,0 +1,65 @@ +DROP TABLE complex_type_nested_staging; +DROP TABLE complex_type_nested; +DROP TABLE fake_table; + +CREATE TABLE if not exists fake_table ( + id int +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|'; + +LOAD DATA LOCAL INPATH '../../data/files/parquet_fake_table.txt' OVERWRITE INTO TABLE fake_table; + +CREATE TABLE if not exists complex_type_nested_staging ( + id int, + array_simple ARRAY, + array_map ARRAY>, + array_array ARRAY>, + array_struct ARRAY>, + map_simple MAP, + map_map MAP>, + map_array MAP>, + map_struct MAP>, + struct_simple STRUCT, + struct_map_array STRUCT, col_array : ARRAY> +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':'; + +INSERT INTO TABLE complex_type_nested_staging +SELECT + 1, + array("1","2","3","4","5"), + array(map('key1', 'value1a','key2', 'value2a'), map('key1', 'value1a','key2', 'value2a'),map('key1', 'value1a','key2', 'value2a')), + array(array("1","2","3","4","5"),array("1","2","3","4","5"),array("1","2","3","4","5"),array("1","2","3","4","5"),array("1","2","3","4","5")), + array(named_struct("col_int", 1, "col_str", "first"), named_struct("col_int", 2, "col_str", "second"), named_struct("col_int", 3, "col_str", "third")), + map('key1', 'value1a','key2', 'value2a'), + map('parentkey_a',map('key1', 'value1a','key2', 'value2a'),'parentkey_b',map('key1', 'value1b','key2', 'value2b')), + map('key1', array("1","2","3","4","5"),'key2', array("1","2","3","4","5")), + map('key1', named_struct("col_int", 1, "col_str", "first"),'key2', named_struct("col_int", 2, "col_str", "second")), + named_struct("col_int", 2, "col_str", "second"), + named_struct("col_map", map('key1', 'value1a','key2', 'value2a'), "col_array", array("1","2","3","4","5")) +FROM + fake_table; + +CREATE TABLE parquet_complex_type_nested ( + id int, + array_simple ARRAY, + array_map ARRAY>, + array_array ARRAY>, + array_struct ARRAY>, + map_simple MAP, + map_map MAP>, + map_array MAP>, + map_struct MAP>, + struct_simple STRUCT, + struct_map_array STRUCT, col_array : ARRAY> +) STORED AS PARQUET; + +DESCRIBE FORMATTED parquet_complex_type_nested; + +INSERT OVERWRITE TABLE parquet_complex_type_nested SELECT * FROM complex_type_nested_staging; + +SELECT * from parquet_complex_type_nested; + + diff --git ql/src/test/results/clientpositive/parquet_complex_type_nested.q.out ql/src/test/results/clientpositive/parquet_complex_type_nested.q.out new file mode 100644 index 0000000..480ad9d --- /dev/null +++ ql/src/test/results/clientpositive/parquet_complex_type_nested.q.out @@ -0,0 +1,271 @@ +PREHOOK: query: DROP TABLE complex_type_nested_staging +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE complex_type_nested_staging +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE complex_type_nested +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE complex_type_nested +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE fake_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE fake_table +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE if not exists fake_table ( + id int +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE if not exists fake_table ( + id int +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@fake_table +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_fake_table.txt' OVERWRITE INTO TABLE fake_table +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@fake_table +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_fake_table.txt' OVERWRITE INTO TABLE fake_table +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@fake_table +PREHOOK: query: CREATE TABLE if not exists complex_type_nested_staging ( + id int, + array_simple ARRAY, + array_map ARRAY>, + array_array ARRAY>, + array_struct ARRAY>, + map_simple MAP, + map_map MAP>, + map_array MAP>, + map_struct MAP>, + struct_simple STRUCT, + struct_map_array STRUCT, col_array : ARRAY> +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE if not exists complex_type_nested_staging ( + id int, + array_simple ARRAY, + array_map ARRAY>, + array_array ARRAY>, + array_struct ARRAY>, + map_simple MAP, + map_map MAP>, + map_array MAP>, + map_struct MAP>, + struct_simple STRUCT, + struct_map_array STRUCT, col_array : ARRAY> +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +COLLECTION ITEMS TERMINATED BY ',' +MAP KEYS TERMINATED BY ':' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@complex_type_nested_staging +PREHOOK: query: INSERT INTO TABLE complex_type_nested_staging +SELECT + 1, + array("1","2","3","4","5"), + array(map('key1', 'value1a','key2', 'value2a'), map('key1', 'value1a','key2', 'value2a'),map('key1', 'value1a','key2', 'value2a')), + array(array("1","2","3","4","5"),array("1","2","3","4","5"),array("1","2","3","4","5"),array("1","2","3","4","5"),array("1","2","3","4","5")), + array(named_struct("col_int", 1, "col_str", "first"), named_struct("col_int", 2, "col_str", "second"), named_struct("col_int", 3, "col_str", "third")), + map('key1', 'value1a','key2', 'value2a'), + map('parentkey_a',map('key1', 'value1a','key2', 'value2a'),'parentkey_b',map('key1', 'value1b','key2', 'value2b')), + map('key1', array("1","2","3","4","5"),'key2', array("1","2","3","4","5")), + map('key1', named_struct("col_int", 1, "col_str", "first"),'key2', named_struct("col_int", 2, "col_str", "second")), + named_struct("col_int", 2, "col_str", "second"), + named_struct("col_map", map('key1', 'value1a','key2', 'value2a'), "col_array", array("1","2","3","4","5")) +FROM + fake_table +PREHOOK: type: QUERY +PREHOOK: Input: default@fake_table +PREHOOK: Output: default@complex_type_nested_staging +POSTHOOK: query: INSERT INTO TABLE complex_type_nested_staging +SELECT + 1, + array("1","2","3","4","5"), + array(map('key1', 'value1a','key2', 'value2a'), map('key1', 'value1a','key2', 'value2a'),map('key1', 'value1a','key2', 'value2a')), + array(array("1","2","3","4","5"),array("1","2","3","4","5"),array("1","2","3","4","5"),array("1","2","3","4","5"),array("1","2","3","4","5")), + array(named_struct("col_int", 1, "col_str", "first"), named_struct("col_int", 2, "col_str", "second"), named_struct("col_int", 3, "col_str", "third")), + map('key1', 'value1a','key2', 'value2a'), + map('parentkey_a',map('key1', 'value1a','key2', 'value2a'),'parentkey_b',map('key1', 'value1b','key2', 'value2b')), + map('key1', array("1","2","3","4","5"),'key2', array("1","2","3","4","5")), + map('key1', named_struct("col_int", 1, "col_str", "first"),'key2', named_struct("col_int", 2, "col_str", "second")), + named_struct("col_int", 2, "col_str", "second"), + named_struct("col_map", map('key1', 'value1a','key2', 'value2a'), "col_array", array("1","2","3","4","5")) +FROM + fake_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@fake_table +POSTHOOK: Output: default@complex_type_nested_staging +POSTHOOK: Lineage: complex_type_nested_staging.array_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_map EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_simple EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_struct EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.id SIMPLE [] +POSTHOOK: Lineage: complex_type_nested_staging.map_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_map EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_simple EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_struct EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.struct_map_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.struct_simple EXPRESSION [] +PREHOOK: query: CREATE TABLE parquet_complex_type_nested ( + id int, + array_simple ARRAY, + array_map ARRAY>, + array_array ARRAY>, + array_struct ARRAY>, + map_simple MAP, + map_map MAP>, + map_array MAP>, + map_struct MAP>, + struct_simple STRUCT, + struct_map_array STRUCT, col_array : ARRAY> +) STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE parquet_complex_type_nested ( + id int, + array_simple ARRAY, + array_map ARRAY>, + array_array ARRAY>, + array_struct ARRAY>, + map_simple MAP, + map_map MAP>, + map_array MAP>, + map_struct MAP>, + struct_simple STRUCT, + struct_map_array STRUCT, col_array : ARRAY> +) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_complex_type_nested +POSTHOOK: Lineage: complex_type_nested_staging.array_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_map EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_simple EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_struct EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.id SIMPLE [] +POSTHOOK: Lineage: complex_type_nested_staging.map_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_map EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_simple EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_struct EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.struct_map_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.struct_simple EXPRESSION [] +PREHOOK: query: DESCRIBE FORMATTED parquet_complex_type_nested +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@parquet_complex_type_nested +POSTHOOK: query: DESCRIBE FORMATTED parquet_complex_type_nested +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@parquet_complex_type_nested +POSTHOOK: Lineage: complex_type_nested_staging.array_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_map EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_simple EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_struct EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.id SIMPLE [] +POSTHOOK: Lineage: complex_type_nested_staging.map_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_map EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_simple EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_struct EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.struct_map_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.struct_simple EXPRESSION [] +# col_name data_type comment + +id int +array_simple array +array_map array> +array_array array> +array_struct array> +map_simple map +map_map map> +map_array map> +map_struct map> +struct_simple struct +struct_map_array struct,col_array:array> + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: INSERT OVERWRITE TABLE parquet_complex_type_nested SELECT * FROM complex_type_nested_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@complex_type_nested_staging +PREHOOK: Output: default@parquet_complex_type_nested +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_complex_type_nested SELECT * FROM complex_type_nested_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@complex_type_nested_staging +POSTHOOK: Output: default@parquet_complex_type_nested +POSTHOOK: Lineage: complex_type_nested_staging.array_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_map EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_simple EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_struct EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.id SIMPLE [] +POSTHOOK: Lineage: complex_type_nested_staging.map_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_map EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_simple EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_struct EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.struct_map_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.struct_simple EXPRESSION [] +POSTHOOK: Lineage: parquet_complex_type_nested.array_array SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:array_array, type:array>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.array_map SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:array_map, type:array>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.array_simple SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:array_simple, type:array, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.array_struct SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:array_struct, type:array>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.id SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.map_array SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:map_array, type:map>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.map_map SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:map_map, type:map>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.map_simple SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:map_simple, type:map, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.map_struct SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:map_struct, type:map>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.struct_map_array SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:struct_map_array, type:struct,col_array:array>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.struct_simple SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:struct_simple, type:struct, comment:null), ] +PREHOOK: query: SELECT * from parquet_complex_type_nested +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_complex_type_nested +#### A masked pattern was here #### +POSTHOOK: query: SELECT * from parquet_complex_type_nested +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_complex_type_nested +#### A masked pattern was here #### +POSTHOOK: Lineage: complex_type_nested_staging.array_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_map EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_simple EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.array_struct EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.id SIMPLE [] +POSTHOOK: Lineage: complex_type_nested_staging.map_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_map EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_simple EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.map_struct EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.struct_map_array EXPRESSION [] +POSTHOOK: Lineage: complex_type_nested_staging.struct_simple EXPRESSION [] +POSTHOOK: Lineage: parquet_complex_type_nested.array_array SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:array_array, type:array>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.array_map SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:array_map, type:array>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.array_simple SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:array_simple, type:array, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.array_struct SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:array_struct, type:array>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.id SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.map_array SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:map_array, type:map>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.map_map SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:map_map, type:map>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.map_simple SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:map_simple, type:map, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.map_struct SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:map_struct, type:map>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.struct_map_array SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:struct_map_array, type:struct,col_array:array>, comment:null), ] +POSTHOOK: Lineage: parquet_complex_type_nested.struct_simple SIMPLE [(complex_type_nested_staging)complex_type_nested_staging.FieldSchema(name:struct_simple, type:struct, comment:null), ] +1 ["1","2","3","4","5"] [{"key1":"value1a","key2":"value2a"},{"key1":"value1a","key2":"value2a"},{"key1":"value1a","key2":"value2a"}] [["1","2","3","4","5"],["1","2","3","4","5"],["1","2","3","4","5"],["1","2","3","4","5"],["1","2","3","4","5"]] [{"col_int":1,"col_str":"first"},{"col_int":2,"col_str":"second"},{"col_int":3,"col_str":"third"}] {"key1":"value1a","key2":"value2a"} {"parentkey_b":{"key1":"value1b","key2":"value2b"},"parentkey_a":{"key1":"value1a","key2":"value2a"}} {"key1":["1","2","3","4","5"],"key2":["1","2","3","4","5"]} {"key1":{"col_int":1,"col_str":"first"},"key2":{"col_int":2,"col_str":"second"}} {"col_int":2,"col_str":"second"} {"col_map":{"key1":"value1a","key2":"value2a"},"col_array":["1","2","3","4","5"]}