diff --git ql/src/test/queries/clientpositive/parquet_schema_evolution.q ql/src/test/queries/clientpositive/parquet_schema_evolution.q index e767b8186ead8e005ef4ee40f583a8afc48fa175..9958d195804ffdec18d4df57a4e572a934461752 100644 --- ql/src/test/queries/clientpositive/parquet_schema_evolution.q +++ ql/src/test/queries/clientpositive/parquet_schema_evolution.q @@ -37,6 +37,39 @@ ALTER TABLE schema_test CHANGE msg msg array PROP_OVERLAY_WHITELIST = + Lists.newArrayList(serdeConstants.LIST_COLUMN_TYPES, + serdeConstants.LIST_COLUMNS, + serdeConstants.SERIALIZATION_DDL); + protected String configErrors; /** @@ -51,8 +59,10 @@ */ public void initialize(Configuration configuration, Properties tableProperties, Properties partitionProperties) throws SerDeException { + initialize(configuration, - SerDeUtils.createOverlayedProperties(tableProperties, partitionProperties)); + SerDeUtils.createOverlayedProperties(tableProperties, partitionProperties, + PROP_OVERLAY_WHITELIST)); } /** diff --git serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java index b9d23dd72580a684788747bfdff4c35b08726fba..b0d79859f528913fb75bf2ee4f4e918ab6ebab24 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java @@ -19,10 +19,13 @@ package org.apache.hadoop.hive.serde2; import java.nio.charset.Charset; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Properties; +import com.google.common.base.Predicates; +import com.google.common.collect.Iterables; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; @@ -517,10 +520,31 @@ public static boolean hasAnyNullObject(Object o, ObjectInspector oi) { * @return the overlayed properties */ public static Properties createOverlayedProperties(Properties tblProps, Properties partProps) { + return createOverlayedProperties(tblProps, partProps, Collections.emptyList()); + } + + /** + * Returns the union of table and partition properties, + * with partition properties taking precedence. + * @param tblProps + * @param partProps + * @param propertyWhiteList + * @return the overlayed properties + */ + public static Properties createOverlayedProperties(Properties tblProps, Properties partProps, + List propertyWhiteList) { Properties props = new Properties(); props.putAll(tblProps); if (partProps != null) { - props.putAll(partProps); + if(propertyWhiteList != null && !propertyWhiteList.isEmpty()) { + for (Map.Entry property : partProps.entrySet()) { + if (!propertyWhiteList.contains(property.getKey())) { + props.put(property.getKey(), property.getValue()); + } + } + } else { + props.putAll(partProps); + } } return props; } diff --git serde/src/test/org/apache/hadoop/hive/serde2/TestSerDeUtils.java serde/src/test/org/apache/hadoop/hive/serde2/TestSerDeUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..04f55a4676f18c9c4d98ff6366d6b8af8dc23084 --- /dev/null +++ serde/src/test/org/apache/hadoop/hive/serde2/TestSerDeUtils.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2; + +import com.google.common.collect.Lists; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Properties; + +/** + * TestSerDeUtils. + * + */ +public class TestSerDeUtils { + + @Test + public void testPropertyOverlaySimple() { + Properties tblProps = new Properties(); + final String key = "Key1"; + tblProps.put(key, "Value1"); + tblProps.put("Key2", "Value3"); + Properties partProps = new Properties(); + partProps.put(key, "Value2"); + partProps.put("Key3", "Value4"); + + Properties results = SerDeUtils.createOverlayedProperties(tblProps, partProps); + Assert.assertTrue(results.containsKey(key)); + Assert.assertTrue(results.containsKey("Key2")); + Assert.assertTrue(results.containsKey("Key3")); + Assert.assertEquals("Value2", results.getProperty(key)); + Assert.assertEquals("Value3", results.getProperty("Key2")); + Assert.assertEquals("Value4", results.getProperty("Key3")); + } + + @Test + public void testPropertyOverlayWhiteList() { + Properties tblProps = new Properties(); + final String key = "Key1"; + tblProps.put(key, "Value1"); + tblProps.put("Key2", "Value3"); + Properties partProps = new Properties(); + partProps.put(key, "Value2"); + partProps.put("Key3", "Value4"); + + Properties results = SerDeUtils.createOverlayedProperties(tblProps, partProps, + Lists.newArrayList(key)); + Assert.assertTrue(results.containsKey(key)); + Assert.assertTrue(results.containsKey("Key2")); + Assert.assertTrue(results.containsKey("Key3")); + Assert.assertEquals("Value1", results.getProperty(key)); + Assert.assertEquals("Value3", results.getProperty("Key2")); + Assert.assertEquals("Value4", results.getProperty("Key3")); + } + + @Test + public void testPropertyOverlayNoPart() { + Properties tblProps = new Properties(); + final String key = "Key1"; + tblProps.put(key, "Value1"); + tblProps.put("Key2", "Value3"); + + Properties results = SerDeUtils.createOverlayedProperties(tblProps, null); + Assert.assertEquals(results, tblProps); + + Properties resultsWhiteList = SerDeUtils.createOverlayedProperties(tblProps, null, + Lists.newArrayList(key)); + Assert.assertEquals(resultsWhiteList, tblProps); + } +}