diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index e25a8cf9a1..9561e7405f 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -247,7 +247,9 @@ public String toString() { ConfVars.SSL_KEYSTORE_PASSWORD.varname, ConfVars.SSL_KEYSTORE_PASSWORD.hiveName, ConfVars.SSL_TRUSTSTORE_PASSWORD.varname, - ConfVars.SSL_TRUSTSTORE_PASSWORD.hiveName + ConfVars.SSL_TRUSTSTORE_PASSWORD.hiveName, + ConfVars.DBACCESS_SSL_TRUSTSTORE_PASSWORD.varname, + ConfVars.DBACCESS_SSL_TRUSTSTORE_PASSWORD.hiveName ); public static ConfVars getMetaConf(String name) { @@ -452,9 +454,19 @@ public static ConfVars getMetaConf(String name) { "Default transaction isolation level for identity generation."), DATANUCLEUS_USE_LEGACY_VALUE_STRATEGY("datanucleus.rdbms.useLegacyNativeValueStrategy", "datanucleus.rdbms.useLegacyNativeValueStrategy", true, ""), - DBACCESS_SSL_PROPS("metastore.dbaccess.ssl.properties", "hive.metastore.dbaccess.ssl.properties", "", - "Comma-separated SSL properties for metastore to access database when JDO connection URL\n" + - "enables SSL access. e.g. javax.net.ssl.trustStore=/tmp/truststore,javax.net.ssl.trustStorePassword=pwd."), + + // Parameters for configuring SSL encryption from the HMS server to the HMS database store + DBACCESS_SSL_TRUSTSTORE_PASSWORD("javax.net.ssl.trustStorePassword", "javax.net.ssl.trustStorePassword", "", + "Password for the Java truststore file."), + DBACCESS_SSL_TRUSTSTORE_PATH("javax.net.ssl.trustStore", "javax.net.ssl.trustStore", "", + "Location of the Java truststore file to use when connecting to the HMS database store. This file consists of \n" + + "a collection of certificates trusted by this application."), + DBACCESS_SSL_TRUSTSTORE_TYPE("javax.net.ssl.trustStoreType", "javax.net.ssl.trustStoreType", "jks", + new StringSetValidator("jceks", "jks", "pkcs12"), "Java truststore file type. Java can read types \n" + + "jceks, jks, and pkcs12. Defaults to jks."), + DBACCESS_USE_SSL("metastore.dbaccess.use.SSL", "hive.metastore.dbaccess.use.SSL", false, + "Set this to true for using SSL encryption from the HMS server to the HMS database store."), + DEFAULTPARTITIONNAME("metastore.default.partition.name", "hive.exec.default.partition.name", "__HIVE_DEFAULT_PARTITION__", "The default partition name in case the dynamic partition column value is null/empty string or any other values that cannot be escaped. \n" + @@ -1070,6 +1082,13 @@ public static ConfVars getMetaConf(String name) { "Deprecated, use METRICS_REPORTERS instead. This configuraiton will be" + " overridden by HIVE_CODAHALE_METRICS_REPORTER_CLASSES and METRICS_REPORTERS if " + "present. Comma separated list of JMX, CONSOLE, JSON_FILE, HADOOP2"), + @Deprecated + DBACCESS_SSL_PROPS("metastore.dbaccess.ssl.properties", "hive.metastore.dbaccess.ssl.properties", "", + "Deprecated. Use the javax.net.ssl.* properties instead. Comma-separated SSL properties for " + + "metastore to access database when JDO connection URL enables SSL access. \n" + + "e.g. javax.net.ssl.trustStore=/tmp/truststore,javax.net.ssl.trustStorePassword=pwd.\n " + + "If both this and the javax.net.ssl.* properties are set, then the latter properties \n" + + "will overwrite what was set in the deprecated property."), // These are all values that we put here just for testing STR_TEST_ENTRY("test.str", "hive.test.str", "defaultval", "comment"), diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index e598a43e4d..04d800335e 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -322,10 +322,65 @@ private static PartitionExpressionProxy createExpressionProxy(Configuration conf } /** - * Configure the SSL properties of the connection from provided config + * Configure SSL encryption to the database store. + * + * The following properties must be set correctly to enable encryption: + * + * 1. metastore.dbaccess.use.SSL (hive.metastore.dbaccess.use.SSL) + * 2. javax.jdo.option.ConnectionURL + * 3. javax.net.ssl.trustStore + * 4. javax.net.ssl.trustStorePassword + * 5. javax.net.ssl.trustStoreType + * + * The last four properties are JSSE (Java) system properties. The Java layer will handle enabling + * encryption once these properties are set. + * + * Additionally, javax.jdo.option.ConnectionURL must have the database-specific SSL flag in the connection URL. + * * @param conf */ private static void configureSSL(Configuration conf) { + configureSSLDangerously(conf); // TODO: Deprecate this method + + boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.DBACCESS_USE_SSL); + LOG.info("Enable SSL to Database Store = {}", useSSL); + + if (useSSL) { + try { + LOG.info("Setting SSL properties to connect to the Database Store"); + String trustStorePath = MetastoreConf.getVar(conf, ConfVars.DBACCESS_SSL_TRUSTSTORE_PATH).trim(); + if (trustStorePath.isEmpty()) { + throw new IllegalArgumentException(ConfVars.DBACCESS_SSL_TRUSTSTORE_PATH.toString() + " Not configured for SSL connection to the Database Store"); + } + // Password should not be clear text + String trustStorePassword = MetastoreConf.getPassword(conf, ConfVars.DBACCESS_SSL_TRUSTSTORE_PASSWORD); + // Already validated in MetaStoreConf + String trustStoreType = MetastoreConf.getVar(conf, ConfVars.DBACCESS_SSL_TRUSTSTORE_TYPE); + + System.setProperty("javax.net.ssl.trustStore", trustStorePath); + System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword); + System.setProperty("javax.net.ssl.trustStoreType", trustStoreType); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to set the SSL properties to connect to the Database Store."); + } + } + } + + /** + * Configure the SSL properties of the connection from provided config + * + * This method was kept for backwards compatibility purposes. + * + * The property metastore.dbaccess.ssl.properties (hive.metastore.dbaccess.ssl.properties) was deprecated in + * HIVE-20992 in favor of more transparent and user-friendly properties. + * + * Please use the javax.net.ssl.* properties instead. Setting those properties will overwrite the values + * of the deprecated property. + * + * @param conf Configuration + */ + @Deprecated + private static void configureSSLDangerously(Configuration conf) { // SSL support String sslPropString = MetastoreConf.getVar(conf, ConfVars.DBACCESS_SSL_PROPS); if (org.apache.commons.lang.StringUtils.isNotEmpty(sslPropString)) { diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index 0cf113c927..7fce9be474 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -68,6 +68,7 @@ import org.junit.Assert; import org.junit.Assume; import org.junit.Before; +import org.junit.After; import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -141,6 +142,14 @@ public void setUp() throws Exception { HiveMetaStore.HMSHandler.createDefaultCatalog(objectStore, new Warehouse(conf)); } + @After + public void tearDown() throws Exception { + // Clear the SSL system properties before each test. + System.clearProperty("javax.net.ssl.trustStore"); + System.clearProperty("javax.net.ssl.trustStorePassword"); + System.clearProperty("javax.net.ssl.trustStoreType"); + } + @Test public void catalogs() throws MetaException, NoSuchObjectException { final String names[] = {"cat1", "cat2"}; @@ -1032,6 +1041,62 @@ public Void call() throws Exception { counter.get()); } + /** + * Test the SSL configuration parameters to ensure that they modify the Java system properties correctly. + */ + @Test + public void testSSLPropertiesAreSet() { + setAndCheckSSLProperties(true, "/tmp/truststore.p12", "password", "pkcs12"); + } + + /** + * Test the property metastore.dbaccess.use.SSL (hive.metastore.dbaccess.use.SSL) to ensure that it correctly + * toggles whether or not the SSL configuration parameters will be set. Effectively, this is testing whether + * SSL can be turned on/off correctly. + */ + @Test + public void testUseSSLProperty() { + setAndCheckSSLProperties(false, "/tmp/truststore.jks", "password", "jks"); + } + + /** + * Test that the deprecated property metastore.dbaccess.ssl.properties is overwritten by the javax.net.ssl.* properties + * if both are set. + * + * This is not an ideal scenario. It is highly recommend to only set the javax.net.ssl.* properties. + */ + @Test + public void testDangerousSSLIsOverwritten() { + // Different from the values in the safe config + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.DBACCESS_SSL_PROPS, + "javax.net.ssl.trustStore=/tmp/truststore.p12,javax.net.ssl.trustStorePassword=pwd,javax.net.ssl.trustStoreType=pkcs12"); + + // Safe config + setAndCheckSSLProperties(true, "/tmp/truststore.jks", "password", "jks"); + } + + /** + * Helper method for setting and checking the SSL configuration parameters. + */ + private void setAndCheckSSLProperties(boolean useSSL, String trustStorePath, String trustStorePassword, String trustStoreType) { + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.DBACCESS_USE_SSL, useSSL); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.DBACCESS_SSL_TRUSTSTORE_PATH, trustStorePath); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.DBACCESS_SSL_TRUSTSTORE_PASSWORD, trustStorePassword); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.DBACCESS_SSL_TRUSTSTORE_TYPE, trustStoreType); + objectStore.setConf(conf); // Calls configureSSL() + + // Check that the Java system values correspond to the values that we set + if (useSSL) { + Assert.assertEquals(trustStorePath, System.getProperty("javax.net.ssl.trustStore")); + Assert.assertEquals(trustStorePassword, System.getProperty("javax.net.ssl.trustStorePassword")); + Assert.assertEquals(trustStoreType, System.getProperty("javax.net.ssl.trustStoreType")); + } else { + Assert.assertNull(System.getProperty("javax.net.ssl.trustStore")); + Assert.assertNull(System.getProperty("javax.net.ssl.trustStorePassword")); + Assert.assertNull(System.getProperty("javax.net.ssl.trustStoreType")); + } + } + private void createTestCatalog(String catName) throws MetaException { Catalog cat = new CatalogBuilder() .setName(catName)