diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java index d826f8be124c447bf54995d417b3bc4927186af5..6c3da970780cfa6d781c8b080a658e3bd419b937 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java @@ -26,10 +26,12 @@ import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hive.common.util.HiveStringUtils; import java.io.File; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -38,6 +40,7 @@ import java.util.Map; import java.util.Set; import java.util.StringTokenizer; +import java.util.stream.Stream; /** * Hive Configuration utils @@ -182,23 +185,37 @@ public static void updateJobCredentialProviders(Configuration jobConf) { String jobKeyStoreLocation = jobConf.get(HiveConf.ConfVars.HIVE_SERVER2_JOB_CREDENTIAL_PROVIDER_PATH.varname); String oldKeyStoreLocation = jobConf.get(Constants.HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG); + if (StringUtils.isNotBlank(jobKeyStoreLocation)) { jobConf.set(Constants.HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG, jobKeyStoreLocation); LOG.debug("Setting job conf credstore location to " + jobKeyStoreLocation + " previous location was " + oldKeyStoreLocation); } - String credStorepassword = getJobCredentialProviderPassword(jobConf); - if (credStorepassword != null) { - // if the execution engine is MR set the map/reduce env with the credential store password + String credstorePassword = getJobCredentialProviderPassword(jobConf); + if (credstorePassword != null) { String execEngine = jobConf.get(ConfVars.HIVE_EXECUTION_ENGINE.varname); + if ("mr".equalsIgnoreCase(execEngine)) { - addKeyValuePair(jobConf, JobConf.MAPRED_MAP_TASK_ENV, - Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, credStorepassword); - addKeyValuePair(jobConf, JobConf.MAPRED_REDUCE_TASK_ENV, - Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, credStorepassword); - addKeyValuePair(jobConf, "yarn.app.mapreduce.am.admin.user.env", - Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, credStorepassword); + // if the execution engine is MR set the map/reduce env with the credential store password + + Collection redactedProperties = + jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES); + + Stream.of( + JobConf.MAPRED_MAP_TASK_ENV, + JobConf.MAPRED_REDUCE_TASK_ENV, + "yarn.app.mapreduce.am.admin.user.env") + + .forEach(property -> { + addKeyValuePair(jobConf, property, + Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, credstorePassword); + redactedProperties.add(property); + }); + + // Hide sensitive configuration values from MR HistoryUI by telling MR to redact the following list. + jobConf.set(MRJobConfig.MR_JOB_REDACTED_PROPERTIES, + StringUtils.join(redactedProperties, ",")); } } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java index 62eb9e4c3c14cfc233f5184d58171e298e25bd9a..4f49190df0c5fecbc05c5af45dfc0e9017d93965 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHiveCredentialProviders.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.exec; import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -27,6 +29,8 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.MRJobConfig; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -44,6 +48,10 @@ private static final String HADOOP_CREDSTORE_LOCATION = "localjceks://file/user/hive/localcreds.jceks"; + private static final Collection REDACTED_PROPERTIES = Arrays.asList( + JobConf.MAPRED_MAP_TASK_ENV, + JobConf.MAPRED_REDUCE_TASK_ENV); + private Configuration jobConf; /* @@ -93,6 +101,9 @@ public void testJobCredentialProvider() throws Exception { // make sure REDUCE task environment points to HIVE_JOB_CREDSTORE_PASSWORD Assert.assertEquals(HIVE_JOB_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .containsAll(REDACTED_PROPERTIES)); } /* @@ -114,6 +125,9 @@ public void testHadoopCredentialProvider() throws Exception { // make sure REDUCE task environment points to HADOOP_CREDSTORE_PASSWORD Assert.assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .containsAll(REDACTED_PROPERTIES)); } /* @@ -131,6 +145,10 @@ public void testNoCredentialProviderWithPassword() throws Exception { Assert.assertNull(getValueFromJobConf(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( + jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .contains(property))); } /* @@ -150,6 +168,9 @@ public void testJobCredentialProviderWithDefaultPassword() throws Exception { Assert.assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .containsAll(REDACTED_PROPERTIES)); } /* @@ -166,6 +187,10 @@ public void testCredentialProviderWithNoPasswords() throws Exception { Assert.assertNull(jobConf.get(JobConf.MAPRED_MAP_TASK_ENV)); Assert.assertNull(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV)); + REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( + jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .contains(property))); + resetConfig(); setupConfigs(true, false, false, false); @@ -174,6 +199,10 @@ public void testCredentialProviderWithNoPasswords() throws Exception { jobConf.get(HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG)); Assert.assertNull(jobConf.get(JobConf.MAPRED_MAP_TASK_ENV)); Assert.assertNull(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV)); + + REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( + jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .contains(property))); } /* @@ -193,6 +222,9 @@ public void testJobCredentialProviderUnset() throws Exception { assertEquals(HADOOP_CREDSTORE_PASSWORD_ENVVAR_VAL, getValueFromJobConf( jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + Assert.assertTrue(jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .containsAll(REDACTED_PROPERTIES)); } /* @@ -210,6 +242,10 @@ public void testNoCredentialProvider() throws Exception { assertNull(getValueFromJobConf(jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV), HADOOP_CREDENTIAL_PASSWORD_ENVVAR)); + + REDACTED_PROPERTIES.forEach(property -> Assert.assertFalse( + jobConf.getStringCollection(MRJobConfig.MR_JOB_REDACTED_PROPERTIES) + .contains(property))); } /*