diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java index ae6fa43df0..bd89c41862 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java @@ -42,6 +42,9 @@ import java.util.StringTokenizer; import java.util.stream.Stream; +import static org.apache.hive.common.util.HiveStringUtils.COMMA; +import static org.apache.hive.common.util.HiveStringUtils.EQUALS; + /** * Hive Configuration utils */ @@ -241,15 +244,17 @@ public static String getJobCredentialProviderPassword(Configuration conf) { return null; } - private static void addKeyValuePair(Configuration jobConf, String property, String keyName, - String newKeyValue) { + private static void addKeyValuePair(Configuration jobConf, String property, String keyName, String newKeyValue) { String existingValue = jobConf.get(property); - if (existingValue == null) { - jobConf.set(property, (keyName + "=" + newKeyValue)); + + if (StringUtils.isBlank(existingValue)) { + jobConf.set(property, (keyName + EQUALS + newKeyValue)); return; } + String propertyValue = existingValue.contains(keyName + EQUALS) ? + HiveStringUtils.insertValue(keyName, newKeyValue, existingValue) : + existingValue + COMMA + (keyName + EQUALS + newKeyValue); - String propertyValue = HiveStringUtils.insertValue(keyName, newKeyValue, existingValue); jobConf.set(property, propertyValue); } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java index 4f49190df0..b4890bc6b5 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java @@ -50,7 +50,8 @@ private static final Collection REDACTED_PROPERTIES = Arrays.asList( JobConf.MAPRED_MAP_TASK_ENV, - JobConf.MAPRED_REDUCE_TASK_ENV); + JobConf.MAPRED_REDUCE_TASK_ENV, + "yarn.app.mapreduce.am.admin.user.env"); private Configuration jobConf; @@ -102,6 +103,9 @@ public void testJobCredentialProvider() throws Exception { Assert.assertEquals(HIVE_JOB_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + Assert.assertEquals(HIVE_JOB_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( + jobConf.get("yarn.app.mapreduce.am.admin.user.env"), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) .containsAll(REDACTED_PROPERTIES)); } @@ -126,6 +130,9 @@ public void testHadoopCredentialProvider() throws Exception { Assert.assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + Assert.assertEquals(HIVE_JOB_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( + jobConf.get("yarn.app.mapreduce.am.admin.user.env"), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) .containsAll(REDACTED_PROPERTIES)); } @@ -146,6 +153,9 @@ public void testNoCredentialProviderWithPassword() throws Exception { Assert.assertNull(getValueFromJobConf(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + Assert.assertNull(getValueFromJobConf(jobConf.get("yarn.app.mapreduce.am.admin.user.env"), + HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) .contains(property))); @@ -169,6 +179,9 @@ public void testJobCredentialProviderWithDefaultPassword() throws Exception { Assert.assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + Assert.assertEquals(HIVE_JOB_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( + jobConf.get("yarn.app.mapreduce.am.admin.user.env"), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) .containsAll(REDACTED_PROPERTIES)); } @@ -186,6 +199,7 @@ public void testCredentialProviderWithNoPasswords() throws Exception { jobConf.get(HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG)); Assert.assertNull(jobConf.get(JobConf.MAPRED_MAP_TASK_ENV)); Assert.assertNull(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV)); + Assert.assertNull(jobConf.get("yarn.app.mapreduce.am.admin.user.env")); REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) @@ -199,6 +213,7 @@ public void testCredentialProviderWithNoPasswords() throws Exception { jobConf.get(HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG)); Assert.assertNull(jobConf.get(JobConf.MAPRED_MAP_TASK_ENV)); Assert.assertNull(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV)); + Assert.assertNull(jobConf.get("yarn.app.mapreduce.am.admin.user.env")); REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) @@ -223,6 +238,9 @@ public void testJobCredentialProviderUnset() throws Exception { assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + Assert.assertEquals(HIVE_JOB_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( + jobConf.get("yarn.app.mapreduce.am.admin.user.env"), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) .containsAll(REDACTED_PROPERTIES)); } @@ -243,6 +261,9 @@ public void testNoCredentialProvider() throws Exception { assertNull(getValueFromJobConf(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + assertNull(getValueFromJobConf(jobConf.get("yarn.app.mapreduce.am.admin.user.env"), + HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) .contains(property)));