diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 056f2d78346b6b306d34dfb610e3a7fed4ca68aa..c3706388350ce67eb4455ec03ce9b99697d66d75 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.conf; import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.FileUtils; @@ -3759,7 +3760,7 @@ public void verifyAndSet(String name, String value) throws IllegalArgumentExcept + "It is not in list of params that are allowed to be modified at runtime"); } } - if (restrictList.contains(name)) { + if (Iterables.any(restrictList, restrictedVar -> restrictedVar.contains(name))) { throw new IllegalArgumentException("Cannot modify " + name + " at runtime. It is in the list" + " of parameters that can't be modified at runtime"); } diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java index 9084fed9fa5121285709874923b3e03e887cb20d..750aff8c5cb748fcd5c3ec99fbc6f335db29448b 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.conf; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -37,6 +39,7 @@ import java.util.Map; import java.util.Set; import java.util.StringTokenizer; +import java.util.function.Consumer; /** * Hive Configuration utils @@ -92,11 +95,20 @@ public static StringBuilder dumpConfig(HiveConf conf) { * @param hiddenSet The values to strip */ public static void stripConfigurations(Configuration conf, Set hiddenSet) { - for (String name : hiddenSet) { - if (conf.get(name) != null) { - conf.set(name, StringUtils.EMPTY); - } - } + + // Find all configurations where the key contains any string from hiddenSet + Iterable> matching = + Iterables.filter(conf, confEntry -> { + for (String name : hiddenSet) { + if (confEntry.getKey().contains(name)) { + return true; + } + } + return false; + }); + + // Remove the value of every key found matching + matching.forEach(entry -> conf.set(entry.getKey(), StringUtils.EMPTY)); } /** diff --git common/src/test/org/apache/hadoop/hive/conf/TestHiveConfRestrictList.java common/src/test/org/apache/hadoop/hive/conf/TestHiveConfRestrictList.java index 980fa761541d72a0b37bc5d687e1fbc4551b5282..adf3cfb1661385b4ec192cf451ddf37709ceee22 100644 --- common/src/test/org/apache/hadoop/hive/conf/TestHiveConfRestrictList.java +++ common/src/test/org/apache/hadoop/hive/conf/TestHiveConfRestrictList.java @@ -41,7 +41,17 @@ protected void setUp() throws Exception { @Test public void testRestriction() throws Exception { verifyRestriction(ConfVars.HIVETESTMODEPREFIX.varname, "foo"); - conf.verifyAndSet(ConfVars.HIVETESTMODE.varname, "false"); + conf.verifyAndSet(ConfVars.HIVE_AM_SPLIT_GENERATION.varname, "false"); + } + + /** + * Test that configs in restrict list can't be changed + * @throws Exception + */ + @Test + public void testMultipleRestrictions() throws Exception { + verifyRestriction(ConfVars.HIVETESTMODEPREFIX.varname, "foo"); + verifyRestriction(ConfVars.HIVETESTMODE.varname, "false"); } /** diff --git common/src/test/org/apache/hadoop/hive/conf/TestHiveConfUntil.java common/src/test/org/apache/hadoop/hive/conf/TestHiveConfUntil.java new file mode 100644 index 0000000000000000000000000000000000000000..ef8ddbad0281785cd11d52dd4b3dfb6485f728da --- /dev/null +++ common/src/test/org/apache/hadoop/hive/conf/TestHiveConfUntil.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.conf; + +import com.google.common.collect.Sets; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * TestHiveConfUntil + * + */ +public class TestHiveConfUntil { + + private HiveConf conf = new HiveConf(); + + @Before + public void init() { + conf.setBoolean("dummyBoolean", true); + conf.set("dummy", "aaa"); + conf.set("dummy2", "aaa"); + conf.set("3dummy", "aaa"); + conf.set("4dummy4", "aaa"); + } + + @Test + public void testHideNonStringVar() throws Exception { + Assert.assertTrue(conf.getBoolean("dummyBoolean", false)); + HiveConfUtil.stripConfigurations(conf, Sets.newHashSet("dummyBoolean")); + Assert.assertFalse(conf.getBoolean("dummyBoolean", false)); + } + + @Test + public void testHideStringVar() throws Exception { + Assert.assertEquals("aaa", conf.get("dummy")); + HiveConfUtil.stripConfigurations(conf, Sets.newHashSet("dummy")); + Assert.assertEquals("", conf.get("dummy")); + } + + @Test + public void testHideMultipleVars() throws Exception { + Assert.assertEquals("aaa", conf.get("dummy")); + Assert.assertEquals("aaa", conf.get("dummy2")); + Assert.assertEquals("aaa", conf.get("3dummy")); + Assert.assertEquals("aaa", conf.get("4dummy4")); + HiveConfUtil.stripConfigurations(conf, Sets.newHashSet("dummy")); + Assert.assertEquals("", conf.get("dummy")); + Assert.assertEquals("", conf.get("dummy2")); + Assert.assertEquals("", conf.get("3dummy")); + Assert.assertEquals("", conf.get("4dummy4")); + } + +}