diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java index 23605a1..368705f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java @@ -127,8 +127,10 @@ private TypeInfo getStructFieldTypeInfo(String field, int fieldIndex) { return hiveFieldTypeInfos.get(i); } } - throw new RuntimeException("cannot find field " + field - + " in " + hiveFieldNames); + //This means hive type doesn't refer this field that comes from file schema. + //i.e. the field is not required for hive table. It can occur due to schema + //evolution where some field is deleted. + return null; } private Converter getFieldConverter(Type type, int fieldIndex, TypeInfo hiveTypeInfo) { diff --git ql/src/test/queries/clientpositive/parquet_type_promotion.q ql/src/test/queries/clientpositive/parquet_type_promotion.q index 46a0f56..c50221d 100644 --- ql/src/test/queries/clientpositive/parquet_type_promotion.q +++ ql/src/test/queries/clientpositive/parquet_type_promotion.q @@ -68,6 +68,11 @@ f2:int,f3:int>>) STORED AS PARQUET; INSERT INTO TABLE arrays_of_struct_to_map select array(named_struct("c1",1,"c2",2)), array(named_struct("f1", 77,"f2",88,"f3",99)) FROM parquet_type_promotion LIMIT 1; SELECT * FROM arrays_of_struct_to_map; +-- Testing schema evolution of dropping column from array> +ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array>, locations2 +array>); +SELECT * FROM arrays_of_struct_to_map; +-- Testing schema evolution of adding columns into array> ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array>, locations2 array>); SELECT * FROM arrays_of_struct_to_map; diff --git ql/src/test/results/clientpositive/parquet_type_promotion.q.out ql/src/test/results/clientpositive/parquet_type_promotion.q.out index dd08631..55f9b27 100644 --- ql/src/test/results/clientpositive/parquet_type_promotion.q.out +++ ql/src/test/results/clientpositive/parquet_type_promotion.q.out @@ -213,12 +213,35 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@arrays_of_struct_to_map #### A masked pattern was here #### [{"c1":1,"c2":2}] [{"f1":77,"f2":88,"f3":99}] -PREHOOK: query: ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array>, locations2 +PREHOOK: query: -- Testing schema evolution of dropping column from array> +ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array>, locations2 +array>) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@arrays_of_struct_to_map +PREHOOK: Output: default@arrays_of_struct_to_map +POSTHOOK: query: -- Testing schema evolution of dropping column from array> +ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array>, locations2 +array>) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@arrays_of_struct_to_map +POSTHOOK: Output: default@arrays_of_struct_to_map +PREHOOK: query: SELECT * FROM arrays_of_struct_to_map +PREHOOK: type: QUERY +PREHOOK: Input: default@arrays_of_struct_to_map +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM arrays_of_struct_to_map +POSTHOOK: type: QUERY +POSTHOOK: Input: default@arrays_of_struct_to_map +#### A masked pattern was here #### +[{"c1":1}] [{"f2":77}] +PREHOOK: query: -- Testing schema evolution of adding columns into array> +ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array>, locations2 array>) PREHOOK: type: ALTERTABLE_REPLACECOLS PREHOOK: Input: default@arrays_of_struct_to_map PREHOOK: Output: default@arrays_of_struct_to_map -POSTHOOK: query: ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array>, locations2 +POSTHOOK: query: -- Testing schema evolution of adding columns into array> +ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array>, locations2 array>) POSTHOOK: type: ALTERTABLE_REPLACECOLS POSTHOOK: Input: default@arrays_of_struct_to_map