From 2bfbb54a7b33277e87144edc1f5a90258d104250 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Mon, 2 Nov 2015 17:00:24 -0800 Subject: [PATCH] HIVE-12320 : hive.metastore.disallow.incompatible.col.type.changes should be true by default --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +- .../org/apache/hadoop/hive/metastore/MetaStoreUtils.java | 13 +++++++++++-- ql/src/test/queries/clientpositive/avro_partitioned.q | 3 ++- ql/src/test/queries/clientpositive/input3.q | 10 ++-------- ql/src/test/queries/clientpositive/orc_int_type_promotion.q | 3 ++- .../test/queries/clientpositive/parquet_schema_evolution.q | 6 +++--- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 3ab73ad..98f9206 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -637,7 +637,7 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "as nulls, so we should set this parameter if we wish to reverse that behaviour. For others, " + "pruning is the correct behaviour"), METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES( - "hive.metastore.disallow.incompatible.col.type.changes", false, + "hive.metastore.disallow.incompatible.col.type.changes", true, "If true (default is false), ALTER TABLE operations which change the type of a\n" + "column (say STRING) to an incompatible type (say MAP) are disallowed.\n" + "RCFile default SerDe (ColumnarSerDe) serializes the values in such a way that the\n" + diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index bbaa1ce..998beda 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -645,8 +645,17 @@ static private boolean areColTypesCompatible(String oldType, String newType) { * Primitive types like INT, STRING, BIGINT, etc are compatible with each other and are * not blocked. */ - if(serdeConstants.PrimitiveTypes.contains(oldType.toLowerCase()) && - serdeConstants.PrimitiveTypes.contains(newType.toLowerCase())) { + boolean oldTypePrimitive = false; + boolean newTypePrimitive = false; + for (String type : serdeConstants.PrimitiveTypes) { + if (oldType.toLowerCase().trim().startsWith(type)) { + oldTypePrimitive = true; + } + if (newType.toLowerCase().trim().startsWith(type)) { + newTypePrimitive = true; + } + } + if(oldTypePrimitive && newTypePrimitive) { return true; } diff --git a/ql/src/test/queries/clientpositive/avro_partitioned.q b/ql/src/test/queries/clientpositive/avro_partitioned.q index a06e7c4..9e6c79a 100644 --- a/ql/src/test/queries/clientpositive/avro_partitioned.q +++ b/ql/src/test/queries/clientpositive/avro_partitioned.q @@ -112,7 +112,7 @@ OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'; -- Insert data into a partition INSERT INTO TABLE episodes_partitioned_serdeproperties PARTITION (doctor_pt) SELECT title, air_date, doctor, doctor as doctor_pt FROM episodes; - +set hive.metastore.disallow.incompatible.col.type.changes=false; -- Evolve the table schema by adding new array field "cast_and_crew" ALTER TABLE episodes_partitioned_serdeproperties SET SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' @@ -144,5 +144,6 @@ WITH SERDEPROPERTIES ('avro.schema.literal'='{ ] }'); +reset hive.metastore.disallow.incompatible.col.type.changes; -- Try selecting from the evolved table SELECT * FROM episodes_partitioned_serdeproperties; diff --git a/ql/src/test/queries/clientpositive/input3.q b/ql/src/test/queries/clientpositive/input3.q index 2efa7a4..1925fff 100644 --- a/ql/src/test/queries/clientpositive/input3.q +++ b/ql/src/test/queries/clientpositive/input3.q @@ -1,7 +1,3 @@ - - - - CREATE TABLE TEST3a(A INT, B DOUBLE) STORED AS TEXTFILE; DESCRIBE TEST3a; CREATE TABLE TEST3b(A ARRAY, B DOUBLE, C MAP) STORED AS TEXTFILE; @@ -16,11 +12,9 @@ ALTER TABLE TEST3b RENAME TO TEST3c; ALTER TABLE TEST3b RENAME TO TEST3c; DESCRIBE TEST3c; SHOW TABLES; +set hive.metastore.disallow.incompatible.col.type.changes=false; EXPLAIN ALTER TABLE TEST3c REPLACE COLUMNS (R1 INT, R2 DOUBLE); ALTER TABLE TEST3c REPLACE COLUMNS (R1 INT, R2 DOUBLE); +reset hive.metastore.disallow.incompatible.col.type.changes; DESCRIBE EXTENDED TEST3c; - - - - diff --git a/ql/src/test/queries/clientpositive/orc_int_type_promotion.q b/ql/src/test/queries/clientpositive/orc_int_type_promotion.q index 4a805a0..908e8d6 100644 --- a/ql/src/test/queries/clientpositive/orc_int_type_promotion.q +++ b/ql/src/test/queries/clientpositive/orc_int_type_promotion.q @@ -50,8 +50,9 @@ select * from alltypes_orc; alter table alltypes_orc change si si bigint; alter table alltypes_orc change i i bigint; select * from alltypes_orc; - +set hive.metastore.disallow.incompatible.col.type.changes=false; alter table alltypes_orc change l l array; +reset hive.metastore.disallow.incompatible.col.type.changes; select * from alltypes_orc; set hive.vectorized.execution.enabled=true; diff --git a/ql/src/test/queries/clientpositive/parquet_schema_evolution.q b/ql/src/test/queries/clientpositive/parquet_schema_evolution.q index af0cf99..d2f2996 100644 --- a/ql/src/test/queries/clientpositive/parquet_schema_evolution.q +++ b/ql/src/test/queries/clientpositive/parquet_schema_evolution.q @@ -11,10 +11,10 @@ INSERT OVERWRITE TABLE NewStructField SELECT named_struct('a1', map('k1','v1'), DESCRIBE NewStructField; SELECT * FROM NewStructField; - +set hive.metastore.disallow.incompatible.col.type.changes=false; -- Adds new fields to the struct types ALTER TABLE NewStructField REPLACE COLUMNS (a struct, a2:struct, a3:int>, b int); - +reset hive.metastore.disallow.incompatible.col.type.changes; DESCRIBE NewStructField; SELECT * FROM NewStructField; @@ -24,4 +24,4 @@ DESCRIBE NewStructFieldTable; SELECT * FROM NewStructFieldTable; DROP TABLE NewStructField; -DROP TABLE NewStructFieldTable; \ No newline at end of file +DROP TABLE NewStructFieldTable; -- 1.7.12.4 (Apple Git-37)