diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 9c15804049..52460e9dcf 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -472,7 +472,12 @@ private static Properties getDataSourceProps(Configuration conf) { String confVal = MetastoreConf.getAsString(conf, var); String varName = var.getVarname(); Object prevVal = prop.setProperty(varName, confVal); - if (MetastoreConf.isPrintable(varName)) { + if (LOG.isDebugEnabled() && MetastoreConf.isPrintable(varName)) { + // The jdbc connection url can contain sensitive information like username and password + // which should be masked out before logging. + if (varName.equals(ConfVars.CONNECT_URL_KEY)) { + confVal = MetaStoreServerUtils.anonymizeConnectionURL(confVal); + } LOG.debug("Overriding {} value {} from jpox.properties with {}", varName, prevVal, confVal); } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java index f3b3866567..da6f63e56e 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java @@ -43,6 +43,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.annotations.VisibleForTesting; @@ -1151,6 +1152,37 @@ public static void setNestedProperty(Object bean, String propertyName, Object va } } + /** + * Mask out all sensitive information from the jdbc connection url string. + * @param connectionURL the connection url, can be null + * @return the anonymized connection url , can be null + */ + public static String anonymizeConnectionURL(String connectionURL) { + if (connectionURL == null) + return null; + connectionURL.trim(); + return maskContent("user", maskContent("password", connectionURL)); + } + + /** + * Replace the value of a sensitive data with asterisks + * @param sensitiveData the data which should be replaced + * @param connectionURL the string containing the sensitive data + */ + private static String maskContent(String sensitiveData, String connectionURL) { + Pattern pattern = Pattern.compile("[;,\\?&]" + sensitiveData + "=((.*?[;,&\\)])|(.*?)$)"); + Matcher matcher = pattern.matcher(connectionURL); + int start = 0; + StringBuffer buffer = new StringBuffer(); + while (matcher.find()) { + buffer.append(connectionURL.substring(start, matcher.start() + 1) + sensitiveData + "=****"); + start = matcher.end() < connectionURL.length() ? matcher.end() - 1 : matcher.end(); + } + buffer.append(connectionURL.substring(start)); + return buffer.toString(); + } + + // ColumnStatisticsObj with info about its db, table, partition (if table is partitioned) public static class ColStatsObjWithSourceInfo { private final ColumnStatisticsObj colStatsObj; diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java index f4bdd734dc..c185324b21 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java @@ -64,6 +64,7 @@ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.filterMapkeys; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -812,5 +813,35 @@ public void testGetPartitionspecsGroupedBySDonePartitionCombined() throws MetaEx assertThat(partition.getSd().getLocation(), is("/a/b")); assertThat(partition.getValues(), is(Collections.singletonList("val2"))); } + + @Test + public void testAnonymizeConnectionURL() { + String connectionURL = null; + String expectedConnectionURL = null; + String result = MetaStoreServerUtils.anonymizeConnectionURL(connectionURL); + assertEquals(expectedConnectionURL, result); + + connectionURL = "jdbc:mysql://localhost:1111/db?user=user&password=password"; + expectedConnectionURL = "jdbc:mysql://localhost:1111/db?user=****&password=****"; + result = MetaStoreServerUtils.anonymizeConnectionURL(connectionURL); + assertEquals(expectedConnectionURL, result); + + connectionURL = "jdbc:derby:sample;user=jill;password=toFetchAPail"; + expectedConnectionURL = "jdbc:derby:sample;user=****;password=****"; + result = MetaStoreServerUtils.anonymizeConnectionURL(connectionURL); + assertEquals(expectedConnectionURL, result); + + connectionURL = "jdbc:mysql://[(host=myhost1,port=1111,user=sandy,password=secret)," + + "(host=myhost2,port=2222,user=finn,password=secret)]/db"; + expectedConnectionURL = "jdbc:mysql://[(host=myhost1,port=1111,user=****,password=****)," + + "(host=myhost2,port=2222,user=****,password=****)]/db"; + result = MetaStoreServerUtils.anonymizeConnectionURL(connectionURL); + assertEquals(expectedConnectionURL, result); + + connectionURL = "jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true"; + result = MetaStoreServerUtils.anonymizeConnectionURL(connectionURL); + assertEquals(connectionURL, result); + } + }