commit 36a46e9ede0a658cab7786fa7424ac324e99795d Author: Vihang Karajgaonkar Date: Fri Mar 24 17:22:20 2017 -0700 HIVE-16297 : Redact configuration entries before dumping diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e12fea1a846c7047dd2e6c18384fc902fb18dd44..bc61426d55e696b0612983adaaba2c3300eb14d5 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4483,7 +4483,7 @@ private void setupRestrictList() { * Strips hidden config entries from configuration */ public void stripHiddenConfigurations(Configuration conf) { - HiveConfUtil.stripConfigurations(conf, hiddenSet); + HiveConfUtil.stripConfigurations(conf, hiddenSet, StringUtils.EMPTY); } /** diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java index 9ba08e532613311853fb1122e756d63f6991b96c..d8a7eb6984c118a57e2073aac9edf7b4bfe886e0 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java @@ -45,6 +45,7 @@ public class HiveConfUtil { private static final String CLASS_NAME = HiveConfUtil.class.getName(); private static final Log LOG = LogFactory.getLog(CLASS_NAME); + public static final String HIVE_CONF_REDACTED_PROPERTY_VALUE = "***REDACTED***"; /** * Check if metastore is being used in embedded mode. * This utility function exists so that the logic for determining the mode is same @@ -90,20 +91,32 @@ public static StringBuilder dumpConfig(HiveConf conf) { * Strips hidden config entries from configuration * @param conf The configuration to strip from * @param hiddenSet The values to strip + * @param replaceString The string value which should be used to replace the values of HiddenSet */ - public static void stripConfigurations(Configuration conf, Set hiddenSet) { + public static void stripConfigurations(Configuration conf, Set hiddenSet, + String replaceString) { for (String name : hiddenSet) { if (conf.get(name) != null) { - conf.set(name, ""); + conf.set(name, replaceString); } } } + /** + * Returns a copy of configuration object from which the configurations provided in the hidden list + * hive.conf.hidden.list are redacted using the replaceString + */ + public static Configuration stripConfigurations(Configuration originalConf, String replaceString) { + Configuration conf = new Configuration(originalConf); + stripConfigurations(conf, getHiddenSet(conf), replaceString); + return conf; + } + public static void dumpConfig(Configuration originalConf, StringBuilder sb) { Set hiddenSet = getHiddenSet(originalConf); sb.append("Values omitted for security reason if present: ").append(hiddenSet).append("\n"); Configuration conf = new Configuration(originalConf); - stripConfigurations(conf, hiddenSet); + stripConfigurations(conf, hiddenSet, StringUtils.EMPTY); Iterator> configIter = conf.iterator(); List> configVals = new ArrayList<>(); diff --git a/common/src/java/org/apache/hive/http/ConfServlet.java b/common/src/java/org/apache/hive/http/ConfServlet.java index 253df4f2d62e5f2397f76a50adf2fb6980873866..11aad39f0a37596c1d1958e262c4ad0ad04f338f 100644 --- a/common/src/java/org/apache/hive/http/ConfServlet.java +++ b/common/src/java/org/apache/hive/http/ConfServlet.java @@ -26,6 +26,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConfUtil; /** * A servlet to print out the running configuration data. @@ -81,11 +82,13 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) * Guts of the servlet - extracted for easy testing. */ static void writeResponse(Configuration conf, Writer out, String format) - throws IOException, BadFormatException { + throws IOException, BadFormatException { + //redact the sensitive information from the configuration values + Configuration hConf = HiveConfUtil.stripConfigurations(conf, HiveConfUtil.HIVE_CONF_REDACTED_PROPERTY_VALUE); if (FORMAT_JSON.equals(format)) { - Configuration.dumpConfiguration(conf, out); + Configuration.dumpConfiguration(hConf, out); } else if (FORMAT_XML.equals(format)) { - conf.writeXml(out); + hConf.writeXml(out); } else { throw new BadFormatException("Bad format: " + format); } diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java index fa51ef64291336453b38f237bca56bd944f2d949..f3df196f0294abc7891b35d412cf13598d7a2bd5 100644 --- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java +++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java @@ -141,6 +141,26 @@ public void testHiddenConfig() throws Exception { } @Test + public void testRedactedConfigs() throws Exception { + HiveConf conf = new HiveConf(); + final String s3key = "fs.s3a.secret.key"; + final String metastorePwd = "testPassword"; + conf.set(HiveConf.ConfVars.METASTOREPWD.varname, metastorePwd); + // check password configs are hidden + Assert.assertTrue(conf.isHiddenConfig(HiveConf.ConfVars.METASTOREPWD.varname)); + Assert.assertTrue(conf.isHiddenConfig(s3key)); + + //check s3key is not set in the config + Assert.assertNull(conf.get(s3key)); + Configuration conf2 = HiveConfUtil.stripConfigurations(conf, HiveConfUtil.HIVE_CONF_REDACTED_PROPERTY_VALUE); + //hidden configs which are set should be redacted + Assert.assertEquals(HiveConfUtil.HIVE_CONF_REDACTED_PROPERTY_VALUE, + conf2.get(HiveConf.ConfVars.METASTOREPWD.varname)); + // Configs should be redacted only when they are present in the original conf + Assert.assertNull(conf2.get(s3key)); + } + + @Test public void testSparkConfigUpdate(){ HiveConf conf = new HiveConf(); Assert.assertFalse(conf.getSparkConfigUpdated()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index a9d03d060adeaa5cad6bef48a63c048f23819d01..ce0bd90d5419d2bbd8a0465c90a0f607385ad3d4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -466,7 +467,10 @@ private void logOutputFormatError(Configuration hconf, HiveException ex) { StringWriter errorWriter = new StringWriter(); errorWriter.append("Failed to create output format; configuration: "); try { - Configuration.dumpConfiguration(hconf, errorWriter); + //redact sensitive information before logging + Configuration.dumpConfiguration( + HiveConfUtil.stripConfigurations(hconf, HiveConfUtil.HIVE_CONF_REDACTED_PROPERTY_VALUE), + errorWriter); } catch (IOException ex2) { errorWriter.append("{ failed to dump configuration: " + ex2.getMessage() + " }"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java index 4c698994e7e970811f68c4123a2eacd1ce158a10..43af29df1a93c67498a4f36897b08fec2eac9059 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java @@ -358,7 +358,9 @@ private void logConfigurations(JobConf localJobConf) { LOG.info("Logging job configuration: "); StringWriter outWriter = new StringWriter(); try { - Configuration.dumpConfiguration(localJobConf, outWriter); + // redact sensitive information before logging + Configuration.dumpConfiguration(HiveConfUtil.stripConfigurations(localJobConf, HiveConfUtil.HIVE_CONF_REDACTED_PROPERTY_VALUE), + outWriter); } catch (IOException e) { LOG.warn("Error logging job configuration", e); }