diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 9c15804049..b61b8fa673 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -53,6 +53,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.jdo.JDOCanRetryException; @@ -472,7 +473,12 @@ private static Properties getDataSourceProps(Configuration conf) { String confVal = MetastoreConf.getAsString(conf, var); String varName = var.getVarname(); Object prevVal = prop.setProperty(varName, confVal); - if (MetastoreConf.isPrintable(varName)) { + if (LOG.isDebugEnabled() && MetastoreConf.isPrintable(varName)) { + // The jdbc connection url can contain sensitive information like username and password + // which should be masked out before logging. + if (varName.equals(ConfVars.CONNECT_URL_KEY)) { + confVal = MetaStoreServerUtils.anonymizeConnectionURL(confVal); + } LOG.debug("Overriding {} value {} from jpox.properties with {}", varName, prevVal, confVal); } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java index f3b3866567..da6f63e56e 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java @@ -43,6 +43,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.annotations.VisibleForTesting; @@ -1151,6 +1152,37 @@ public static void setNestedProperty(Object bean, String propertyName, Object va } } + /** + * Mask out all sensitive information from the jdbc connection url string. + * @param connectionURL the connection url, can be null + * @return the anonymized connection url , can be null + */ + public static String anonymizeConnectionURL(String connectionURL) { + if (connectionURL == null) + return null; + connectionURL.trim(); + return maskContent("user", maskContent("password", connectionURL)); + } + + /** + * Replace the value of a sensitive data with asterisks + * @param sensitiveData the data which should be replaced + * @param connectionURL the string containing the sensitive data + */ + private static String maskContent(String sensitiveData, String connectionURL) { + Pattern pattern = Pattern.compile("[;,\\?&]" + sensitiveData + "=((.*?[;,&\\)])|(.*?)$)"); + Matcher matcher = pattern.matcher(connectionURL); + int start = 0; + StringBuffer buffer = new StringBuffer(); + while (matcher.find()) { + buffer.append(connectionURL.substring(start, matcher.start() + 1) + sensitiveData + "=****"); + start = matcher.end() < connectionURL.length() ? matcher.end() - 1 : matcher.end(); + } + buffer.append(connectionURL.substring(start)); + return buffer.toString(); + } + + // ColumnStatisticsObj with info about its db, table, partition (if table is partitioned) public static class ColStatsObjWithSourceInfo { private final ColumnStatisticsObj colStatsObj; diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java index f4bdd734dc..543b82d59d 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreServerUtils.java @@ -65,6 +65,7 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -812,5 +813,35 @@ public void testGetPartitionspecsGroupedBySDonePartitionCombined() throws MetaEx assertThat(partition.getSd().getLocation(), is("/a/b")); assertThat(partition.getValues(), is(Collections.singletonList("val2"))); } + + @Test + public void testAnonymizeConnectionURL() { + String connectionURL = null; + String result = MetaStoreServerUtils.anonymizeConnectionURL(connectionURL); + assertNull(result); + + connectionURL = "jdbc:mysql://localhost:1111/db?user=user&password=password"; + result = MetaStoreServerUtils.anonymizeConnectionURL(connectionURL); + assertTrue(result.contains("user=****") && result.contains("password=****")); + assertFalse(result.contains("user=user") || result.contains("password=password")); + + connectionURL = "jdbc:derby:sample;user=jill;password=toFetchAPail"; + result = MetaStoreServerUtils.anonymizeConnectionURL(connectionURL); + assertTrue(result.contains("user=****") && result.contains("password=****")); + assertFalse(result.contains("user=jill") || result.contains("password=toFetchAPail")); + + connectionURL = "jdbc:mysql://[(host=myhost1,port=1111,user=sandy,password=secret)," + + "(host=myhost2,port=2222,user=finn,password=secret)]/db"; + result = MetaStoreServerUtils.anonymizeConnectionURL(connectionURL); + assertTrue(result.contains("user=****") && result.contains("password=****")); + assertFalse(result.contains("user=sandy") || result.contains("password=secret") + || result.contains("user=finn")); + + connectionURL = "jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true"; + result = MetaStoreServerUtils.anonymizeConnectionURL(connectionURL); + assertFalse(result.contains("user=****") || result.contains("password=****")); + assertFalse(result.contains("user=") || result.contains("password=")); + } + }