diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index d50912b4e2..7f40d7de35 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2672,6 +2672,21 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal new TimeValidator(TimeUnit.MILLISECONDS), "Initial amount of time (in milliseconds) to wait between retries\n" + "when connecting to the ZooKeeper server when using ExponentialBackoffRetry policy."), + HIVE_ZOOKEEPER_SSL_ENABLE("hive.zookeeper.ssl.client.enable", false, + "Set client to use TLS when connecting to ZooKeeper. An explicit value overrides any value set via the " + + "zookeeper.client.secure system property (note the different name). Defaults to false if neither is set."), + HIVE_ZOOKEEPER_SSL_KEYSTORE_LOCATION("hive.zookeeper.ssl.keystore.location","", + "Keystore location when using a client-side certificate with TLS connectivity to ZooKeeper. " + + "Overrides any explicit value set via the zookeeper.ssl.keyStore.location system property (note the camelCase)."), + HIVE_ZOOKEEPER_SSL_KEYSTORE_PASSWORD("hive.zookeeper.ssl.keystore.password","", + "Keystore password when using a client-side certificate with TLS connectivity to ZooKeeper." + + "Overrides any explicit value set via the zookeeper.ssl.keyStore.password system property (note the camelCase)."), + HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION("hive.zookeeper.ssl.truststore.location","", + "Truststore location when using a client-side certificate with TLS connectivity to ZooKeeper. " + + "Overrides any explicit value set via the zookeeper.ssl.trustStore.location system property (note the camelCase)."), + HIVE_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD("hive.zookeeper.ssl.truststore.password","", + "Truststore password when using a client-side certificate with TLS connectivity to ZooKeeper." + + "Overrides any explicit value set via the zookeeper.ssl.trustStore.password system property (note the camelCase)."), // Transactions HIVE_TXN_MANAGER("hive.txn.manager", @@ -5620,13 +5635,17 @@ public void logVars(PrintStream ps) { */ public ZooKeeperHiveHelper getZKConfig() { return new ZooKeeperHiveHelper(getVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_QUORUM), - getVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT), - getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE), - (int) getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, - TimeUnit.MILLISECONDS), - (int) getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_BASESLEEPTIME, - TimeUnit.MILLISECONDS), - getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_MAX_RETRIES)); + getVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT), + getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE), + (int) getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS), + (int) getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS), + (int) getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_BASESLEEPTIME, TimeUnit.MILLISECONDS), + getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_MAX_RETRIES), + getBoolVar(ConfVars.HIVE_ZOOKEEPER_SSL_ENABLE), + getVar(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_LOCATION), + getVar(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_PASSWORD), + getVar(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION), + getVar(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD)); } public HiveConf() { diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java index 1fc8d36394..ed2e2135e4 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.common.SSLZookeeperFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.curator.framework.CuratorFramework; @@ -50,8 +51,12 @@ public String overhead_path = null; public static final String ZK_HOSTS = "templeton.zookeeper.hosts"; - public static final String ZK_SESSION_TIMEOUT - = "templeton.zookeeper.session-timeout"; + public static final String ZK_SESSION_TIMEOUT = "templeton.zookeeper.session-timeout"; + public static final String ZK_SSL_ENABLE = "templeton.zookeeper.ssl.client.enable"; + public static final String ZK_KEYSTORE_LOCATION = "templeton.zookeeper.keystore.location"; + public static final String ZK_KEYSTORE_PASSWORD = "templeton.zookeeper.keystore.password"; + public static final String ZK_TRUSTSTORE_LOCATION = "templeton.zookeeper.truststore.location"; + public static final String ZK_TRUSTSTORE_PASSWORD = "templeton.zookeeper.truststore.password"; public static final String ENCODING = "UTF-8"; @@ -59,29 +64,32 @@ private CuratorFramework zk; + /** * Open a ZooKeeper connection for the JobState. */ - public static CuratorFramework zkOpen(String zkHosts, int zkSessionTimeoutMs) - throws IOException { - //do we need to add a connection status listener? What will that do? + public static CuratorFramework zkOpen(Configuration conf) throws IOException { + + String zkHosts = conf.get(ZK_HOSTS); + /* the silly looking call to Builder below is to get the default value of session timeout + from Curator which itself exposes it as system property */ + int sessionTimeoutMs = conf.getInt(ZK_SESSION_TIMEOUT, CuratorFrameworkFactory.builder().getSessionTimeoutMs()); + boolean sslEnabled = conf.getBoolean(ZK_SSL_ENABLE, false); + String keyStoreLocation = conf.get(ZK_KEYSTORE_LOCATION, ""); + String keyStorePassword = conf.get(ZK_KEYSTORE_PASSWORD, ""); + String trustStoreLocation = conf.get(ZK_TRUSTSTORE_LOCATION, ""); + String trustStorePassword = conf.get(ZK_TRUSTSTORE_PASSWORD, ""); ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3); - CuratorFramework zk = CuratorFrameworkFactory.newClient(zkHosts, zkSessionTimeoutMs, - CuratorFrameworkFactory.builder().getConnectionTimeoutMs(), retryPolicy); + CuratorFramework zk = CuratorFrameworkFactory.builder() + .connectString(zkHosts) + .sessionTimeoutMs(sessionTimeoutMs) + .retryPolicy(retryPolicy) + .zookeeperFactory(new SSLZookeeperFactory(sslEnabled,keyStoreLocation,keyStorePassword,trustStoreLocation,trustStorePassword)) + .build(); zk.start(); return zk; } - /** - * Open a ZooKeeper connection for the JobState. - */ - public static CuratorFramework zkOpen(Configuration conf) throws IOException { - /*the silly looking call to Builder below is to get the default value of session timeout - from Curator which itself exposes it as system property*/ - return zkOpen(conf.get(ZK_HOSTS), - conf.getInt(ZK_SESSION_TIMEOUT, CuratorFrameworkFactory.builder().getSessionTimeoutMs())); - } - public ZooKeeperStorage() { // No-op -- this is needed to be able to instantiate the // class from the name. diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZooKeeperTokenStore.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZooKeeperTokenStore.java index 603155bf8f..96b2df5181 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZooKeeperTokenStore.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZooKeeperTokenStore.java @@ -25,11 +25,9 @@ import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge.Server.ServerMode; -import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; import org.apache.hadoop.security.token.delegation.HiveDelegationTokenSupport; @@ -52,28 +50,31 @@ */ public class TestZooKeeperTokenStore { + private static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks"; + private static final String TRUST_STORE_NAME = "truststore.jks"; + private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc"; + private MiniZooKeeperCluster zkCluster = null; - private CuratorFramework zkClient = null; private int zkPort = -1; private ZooKeeperTokenStore ts; + private boolean sslEnabled; @Before public void setUp() throws Exception { + setUpInternal(false); + } + public void setUpInternal(boolean sslEnabled) throws Exception{ File zkDataDir = new File(System.getProperty("test.tmp.dir")); if (this.zkCluster != null) { throw new IOException("Cluster already running"); } - this.zkCluster = new MiniZooKeeperCluster(); + this.zkCluster = new MiniZooKeeperCluster(sslEnabled); this.zkPort = this.zkCluster.startup(zkDataDir); - this.zkClient = - CuratorFrameworkFactory.builder().connectString("localhost:" + zkPort) - .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build(); - this.zkClient.start(); + this.sslEnabled = sslEnabled; } @After - public void tearDown() throws Exception { - this.zkClient.close(); + public void tearDown() throws Exception{ if (ts != null) { ts.close(); } @@ -86,6 +87,17 @@ private Configuration createConf(String zkPath) { conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR, "localhost:" + this.zkPort); conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ZNODE, zkPath); + if(sslEnabled) { + String dataFileDir = !System.getProperty("test.data.files", "").isEmpty() ? + System.getProperty("test.data.files") : + (new HiveConf()).get("test.data.files").replace('\\', '/').replace("c:", ""); + conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_LOCATION, dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); + conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_PASSWORD, KEY_STORE_TRUST_STORE_PASSWORD); + conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_LOCATION, dataFileDir + File.separator + TRUST_STORE_NAME); + conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_PASSWORD, KEY_STORE_TRUST_STORE_PASSWORD); + conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_SSL_ENABLE, "true"); + + } return conf; } @@ -98,6 +110,7 @@ public void testTokenStorage() throws Exception { ts.setConf(conf); ts.init(null, HadoopThriftAuthBridge.Server.ServerMode.METASTORE); + CuratorFramework zkClient = ts.getSession(); String metastore_zk_path = ZK_PATH + ServerMode.METASTORE; int keySeq = ts.addMasterKey("key1Data"); @@ -112,6 +125,7 @@ public void testTokenStorage() throws Exception { ts.removeMasterKey(keySeq); assertEquals("expected number keys", 1, ts.getMasterKeys().length); + ts.removeMasterKey(keySeq2); // tokens DelegationTokenIdentifier tokenId = new DelegationTokenIdentifier( @@ -189,6 +203,8 @@ public void testAclPositive() throws Exception { ts = new ZooKeeperTokenStore(); ts.setConf(conf); ts.init(null, HadoopThriftAuthBridge.Server.ServerMode.METASTORE); + + CuratorFramework zkClient = ts.getSession(); List acl = zkClient.getACL().forPath(ZK_PATH + ServerMode.METASTORE); assertEquals(2, acl.size()); } diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZookeeperTokenStoreSSLEnabled.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZookeeperTokenStoreSSLEnabled.java new file mode 100644 index 0000000000..e1f99e6f01 --- /dev/null +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZookeeperTokenStoreSSLEnabled.java @@ -0,0 +1,16 @@ +package org.apache.hadoop.hive.metastore.security; + +import org.junit.Before; + +public class TestZookeeperTokenStoreSSLEnabled extends TestZooKeeperTokenStore { + + public TestZookeeperTokenStoreSSLEnabled(){ + super(); + } + + @Override + @Before + public void setUp() throws Exception { + setUpInternal(true); + } +} diff --git itests/hive-unit/src/test/java/org/apache/hive/service/server/TestInformationSchemaWithPrivilege.java itests/hive-unit/src/test/java/org/apache/hive/service/server/TestInformationSchemaWithPrivilege.java index de2e4937a8..538c8ca98d 100644 --- itests/hive-unit/src/test/java/org/apache/hive/service/server/TestInformationSchemaWithPrivilege.java +++ itests/hive-unit/src/test/java/org/apache/hive/service/server/TestInformationSchemaWithPrivilege.java @@ -55,7 +55,7 @@ import org.apache.hive.service.cli.RowSet; import org.apache.hive.service.cli.SessionHandle; import org.junit.Assert; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; /** @@ -175,14 +175,22 @@ public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreC } } - private static MiniHS2 miniHS2 = null; - private static MiniZooKeeperCluster zkCluster = null; - private static Map confOverlay; + private static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks"; + private static final String TRUST_STORE_NAME = "truststore.jks"; + private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc"; - @BeforeClass - public static void beforeTest() throws Exception { + private MiniHS2 miniHS2 = null; + private MiniZooKeeperCluster zkCluster = null; + private Map confOverlay; + + @Before + public void setUp() throws Exception { + setupInternal(false); + } + + protected void setupInternal(boolean zookeeperSSLEnabled) throws Exception { File zkDataDir = new File(System.getProperty("test.tmp.dir")); - zkCluster = new MiniZooKeeperCluster(); + zkCluster = new MiniZooKeeperCluster(zookeeperSSLEnabled); int zkPort = zkCluster.startup(zkDataDir); miniHS2 = new MiniHS2(new HiveConf()); @@ -206,6 +214,16 @@ public static void beforeTest() throws Exception { confOverlay.put(ConfVars.HIVE_AUTHENTICATOR_MANAGER.varname, FakeGroupAuthenticator.class.getName()); confOverlay.put(ConfVars.HIVE_AUTHORIZATION_ENABLED.varname, "true"); confOverlay.put(ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST.varname, ".*"); + + if(zookeeperSSLEnabled) { + String dataFileDir = !System.getProperty("test.data.files", "").isEmpty() ? System.getProperty("test.data.files") : (new HiveConf()).get("test.data.files").replace('\\', '/') + .replace("c:", ""); + confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_LOCATION.varname, dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); + confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_PASSWORD.varname, KEY_STORE_TRUST_STORE_PASSWORD); + confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION.varname, dataFileDir + File.separator + TRUST_STORE_NAME); + confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD.varname, KEY_STORE_TRUST_STORE_PASSWORD); + confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_ENABLE.varname, "true"); + } miniHS2.start(confOverlay); } diff --git itests/hive-unit/src/test/java/org/apache/hive/service/server/TestInformationSchemaWithPriviligeZookeeperSSL.java itests/hive-unit/src/test/java/org/apache/hive/service/server/TestInformationSchemaWithPriviligeZookeeperSSL.java new file mode 100644 index 0000000000..ac845028e1 --- /dev/null +++ itests/hive-unit/src/test/java/org/apache/hive/service/server/TestInformationSchemaWithPriviligeZookeeperSSL.java @@ -0,0 +1,16 @@ +package org.apache.hive.service.server; + +import org.junit.Before; + +public class TestInformationSchemaWithPriviligeZookeeperSSL extends TestInformationSchemaWithPrivilege { + + public TestInformationSchemaWithPriviligeZookeeperSSL() { + super(); + } + + @Override + @Before + public void setUp() throws Exception{ + setupInternal(true); + } +} diff --git jdbc/src/java/org/apache/hive/jdbc/Utils.java jdbc/src/java/org/apache/hive/jdbc/Utils.java index e23826eb56..6cb6853077 100644 --- jdbc/src/java/org/apache/hive/jdbc/Utils.java +++ jdbc/src/java/org/apache/hive/jdbc/Utils.java @@ -116,6 +116,11 @@ public static final String SERVICE_DISCOVERY_MODE_ZOOKEEPER = "zooKeeper"; public static final String SERVICE_DISCOVERY_MODE_ZOOKEEPER_HA = "zooKeeperHA"; public static final String ZOOKEEPER_NAMESPACE = "zooKeeperNamespace"; + public static final String ZOOKEEPER_SSL_ENABLE = "zooKeeperSSLEnable"; + public static final String ZOOKEEPER_KEYSTORE_LOCATION = "zooKeeperKeystoreLocation"; + public static final String ZOOKEEPER_KEYSTORE_PASSWORD= "zooKeeperKeystorePassword"; + public static final String ZOOKEEPER_TRUSTSTORE_LOCATION = "zooKeeperTruststoreLocation"; + public static final String ZOOKEEPER_TRUSTSTORE_PASSWORD = "zooKeeperTruststorePassword"; // Default namespace value on ZooKeeper. // This value is used if the param "zooKeeperNamespace" is not specified in the JDBC Uri. static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2"; @@ -168,6 +173,11 @@ private boolean isEmbeddedMode = false; private String suppliedURLAuthority; private String zooKeeperEnsemble = null; + private boolean zooKeeperSslEnabled = false; + private String zookeeperKeyStoreLocation = ""; + private String zookeeperKeyStorePassword = ""; + private String zookeeperTrustStoreLocation = ""; + private String zookeeperTrustStorePassword = ""; private String currentHostZnodePath; private final List rejectedHostZnodePaths = new ArrayList(); @@ -185,6 +195,12 @@ public JdbcConnectionParams(JdbcConnectionParams params) { this.isEmbeddedMode = params.isEmbeddedMode; this.suppliedURLAuthority = params.suppliedURLAuthority; this.zooKeeperEnsemble = params.zooKeeperEnsemble; + this.zooKeeperSslEnabled = params.zooKeeperSslEnabled; + this.zookeeperKeyStoreLocation = params.zookeeperKeyStoreLocation; + this.zookeeperKeyStorePassword = params.zookeeperKeyStorePassword; + this.zookeeperTrustStoreLocation = params.zookeeperTrustStoreLocation; + this.zookeeperTrustStorePassword = params.zookeeperTrustStorePassword; + this.currentHostZnodePath = params.currentHostZnodePath; this.rejectedHostZnodePaths.addAll(rejectedHostZnodePaths); } @@ -228,6 +244,25 @@ public String getSuppliedURLAuthority() { public String getZooKeeperEnsemble() { return zooKeeperEnsemble; } + public boolean isZooKeeperSslEnabled() { + return zooKeeperSslEnabled; + } + + public String getZookeeperKeyStoreLocation() { + return zookeeperKeyStoreLocation; + } + + public String getZookeeperKeyStorePassword() { + return zookeeperKeyStorePassword; + } + + public String getZookeeperTrustStoreLocation() { + return zookeeperTrustStoreLocation; + } + + public String getZookeeperTrustStorePassword() { + return zookeeperTrustStorePassword; + } public List getRejectedHostZnodePaths() { return rejectedHostZnodePaths; @@ -277,6 +312,26 @@ public void setZooKeeperEnsemble(String zooKeeperEnsemble) { this.zooKeeperEnsemble = zooKeeperEnsemble; } + public void setZooKeeperSslEnabled(boolean zooKeeperSslEnabled) { + this.zooKeeperSslEnabled = zooKeeperSslEnabled; + } + + public void setZookeeperKeyStoreLocation(String zookeeperKeyStoreLocation) { + this.zookeeperKeyStoreLocation = zookeeperKeyStoreLocation; + } + + public void setZookeeperKeyStorePassword(String zookeeperKeyStorePassword) { + this.zookeeperKeyStorePassword = zookeeperKeyStorePassword; + } + + public void setZookeeperTrustStoreLocation(String zookeeperTrustStoreLocation) { + this.zookeeperTrustStoreLocation = zookeeperTrustStoreLocation; + } + + public void setZookeeperTrustStorePassword(String zookeeperTrustStorePassword) { + this.zookeeperTrustStorePassword = zookeeperTrustStorePassword; + } + public void setCurrentHostZnodePath(String currentHostZnodePath) { this.currentHostZnodePath = currentHostZnodePath; } @@ -485,6 +540,7 @@ public static JdbcConnectionParams extractURLComponents(String uri, Properties i uri = uri.replace(dummyAuthorityString, authorityStr); // Set ZooKeeper ensemble in connParams for later use connParams.setZooKeeperEnsemble(authorityStr); + ZooKeeperHiveClientHelper.setZkSSLParams(connParams); } else { URI jdbcBaseURI = URI.create(URI_HIVE_PREFIX + "//" + authorityStr); // Check to prevent unintentional use of embedded mode. A missing "/" @@ -576,7 +632,6 @@ private static void handleParamDeprecation(Map fromMap, Map sessionConf) JdbcConnectionParams.SERVICE_DISCOVERY_MODE_ZOOKEEPER_HA.equalsIgnoreCase(discoveryMode)); } + /** + * Parse and set up the SSL communication related Zookeeper params in connParams from sessionVars + * @param connParams + */ + public static void setZkSSLParams(JdbcConnectionParams connParams) { + Map sessionConf = connParams.getSessionVars(); + boolean sslEnabled = false; + if (sessionConf.containsKey(JdbcConnectionParams.ZOOKEEPER_SSL_ENABLE)) { + sslEnabled = Boolean.parseBoolean(sessionConf.get(JdbcConnectionParams.ZOOKEEPER_SSL_ENABLE)); + connParams.setZooKeeperSslEnabled(sslEnabled); + } + if (sslEnabled) { + connParams.setZookeeperKeyStoreLocation( + StringUtils.defaultString(sessionConf.get(JdbcConnectionParams.ZOOKEEPER_KEYSTORE_LOCATION), "")); + connParams.setZookeeperKeyStorePassword( + StringUtils.defaultString(sessionConf.get(JdbcConnectionParams.ZOOKEEPER_KEYSTORE_PASSWORD), "")); + connParams.setZookeeperTrustStoreLocation( + StringUtils.defaultString(sessionConf.get(JdbcConnectionParams.ZOOKEEPER_TRUSTSTORE_LOCATION), "")); + connParams.setZookeeperTrustStorePassword( + StringUtils.defaultString(sessionConf.get(JdbcConnectionParams.ZOOKEEPER_TRUSTSTORE_PASSWORD), "")); + } + } + private static CuratorFramework getZkClient(JdbcConnectionParams connParams) throws Exception { String zooKeeperEnsemble = connParams.getZooKeeperEnsemble(); CuratorFramework zooKeeperClient = - CuratorFrameworkFactory.builder().connectString(zooKeeperEnsemble) - .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build(); + CuratorFrameworkFactory.builder() + .connectString(zooKeeperEnsemble) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .zookeeperFactory( + new SSLZookeeperFactory(connParams.isZooKeeperSslEnabled(), connParams.getZookeeperKeyStoreLocation(), + connParams.getZookeeperKeyStorePassword(), connParams.getZookeeperTrustStoreLocation(), + connParams.getZookeeperTrustStorePassword())) + .build(); zooKeeperClient.start(); return zooKeeperClient; } diff --git llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java index d28fd1778c..88305b6359 100644 --- llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java +++ llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java @@ -44,6 +44,7 @@ import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.curator.utils.CloseableUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.SSLZookeeperFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.LlapUtil; @@ -220,6 +221,12 @@ private CuratorFramework getZookeeperClient(Configuration conf, String namespace int baseSleepTime = (int) HiveConf.getTimeVar(conf, ConfVars.HIVE_ZOOKEEPER_CONNECTION_BASESLEEPTIME, TimeUnit.MILLISECONDS); int maxRetries = HiveConf.getIntVar(conf, ConfVars.HIVE_ZOOKEEPER_CONNECTION_MAX_RETRIES); + boolean sslEnabled = HiveConf.getBoolVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_ENABLE); + String keyStoreLocation = HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_LOCATION); + String keyStorePassword = HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_PASSWORD); + String trustStoreLocation = HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION); + String trustStorePassword = HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD); + LOG.info("Creating curator client with connectString: {} sessionTimeoutMs: {} connectionTimeoutMs: {}" + " namespace: {} exponentialBackoff - sleepTime: {} maxRetries: {}", zkEnsemble, sessionTimeout, @@ -233,6 +240,7 @@ private CuratorFramework getZookeeperClient(Configuration conf, String namespace .aclProvider(zooKeeperAclProvider) .namespace(namespace) .retryPolicy(new ExponentialBackoffRetry(baseSleepTime, maxRetries)) + .zookeeperFactory(new SSLZookeeperFactory(sslEnabled,keyStoreLocation,keyStorePassword,trustStoreLocation,trustStorePassword)) .build(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/CuratorFrameworkSingleton.java ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/CuratorFrameworkSingleton.java index fa3a382367..4338e64fba 100644 --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/CuratorFrameworkSingleton.java +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/CuratorFrameworkSingleton.java @@ -18,14 +18,11 @@ package org.apache.hadoop.hive.ql.lockmgr.zookeeper; -import java.util.concurrent.TimeUnit; import org.apache.hive.common.util.ShutdownHookManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.hive.conf.HiveConf; public class CuratorFrameworkSingleton { @@ -50,15 +47,7 @@ public static synchronized CuratorFramework getInstance(HiveConf hiveConf) { } else { conf = hiveConf; } - int sessionTimeout = (int) conf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS); - int baseSleepTime = (int) conf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_BASESLEEPTIME, TimeUnit.MILLISECONDS); - int maxRetries = conf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_MAX_RETRIES); - String quorumServers = conf.getZKConfig().getQuorumServers(); - - sharedClient = CuratorFrameworkFactory.builder().connectString(quorumServers) - .sessionTimeoutMs(sessionTimeout) - .retryPolicy(new ExponentialBackoffRetry(baseSleepTime, maxRetries)) - .build(); + sharedClient = hiveConf.getZKConfig().getNewZookeeperClient(null); sharedClient.start(); } diff --git ql/src/test/org/apache/hive/testutils/MiniZooKeeperCluster.java ql/src/test/org/apache/hive/testutils/MiniZooKeeperCluster.java index eec628263a..dca2368b58 100644 --- ql/src/test/org/apache/hive/testutils/MiniZooKeeperCluster.java +++ ql/src/test/org/apache/hive/testutils/MiniZooKeeperCluster.java @@ -34,7 +34,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; -import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.common.ClientX509Util; +import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.server.ServerCnxnFactory; import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.persistence.FileTxnLog; import org.slf4j.Logger; @@ -54,6 +56,9 @@ private static final int TICK_TIME = 2000; private static final int DEFAULT_CONNECTION_TIMEOUT = 30000; + private static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks"; + private static final String TRUST_STORE_NAME = "truststore.jks"; + private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc"; private int connectionTimeout; private boolean started; @@ -61,7 +66,7 @@ /** The default port. If zero, we use a random port. */ private int defaultClientPort = 0; - private List standaloneServerFactoryList; + private List standaloneServerFactoryList; private List zooKeeperServers; private List clientPortList; @@ -70,11 +75,20 @@ private Configuration configuration; + private boolean sslEnabled = false; + public MiniZooKeeperCluster() { this(new Configuration()); } - + public MiniZooKeeperCluster(boolean sslEnabled) { + this(new Configuration(), sslEnabled); + } public MiniZooKeeperCluster(Configuration configuration) { + this(configuration, false); + } + + public MiniZooKeeperCluster(Configuration configuration, boolean sslEnabled) { + this.sslEnabled = sslEnabled; this.started = false; this.configuration = configuration; activeZKServerIndex = -1; @@ -176,6 +190,7 @@ private static void setupTestEnv() { // resulting in test failure (client timeout on first session). // set env and directly in order to handle static init/gc issues System.setProperty("zookeeper.preAllocSize", "100"); + System.setProperty("zookeeper.authProvider.x509", "org.apache.zookeeper.server.auth.X509AuthenticationProvider"); FileTxnLog.setPreallocSize(100 * 1024); } @@ -229,12 +244,28 @@ public int startup(File baseDir, int numZooKeeperServers) throws IOException, In // Setting {min,max}SessionTimeout defaults to be the same as in Zookeeper server.setMinSessionTimeout(configuration.getInt("hbase.zookeeper.property.minSessionTimeout", -1)); server.setMaxSessionTimeout(configuration.getInt("hbase.zookeeper.property.maxSessionTimeout", -1)); - NIOServerCnxnFactory standaloneServerFactory; + ServerCnxnFactory standaloneServerFactory; while (true) { try { - standaloneServerFactory = new NIOServerCnxnFactory(); - standaloneServerFactory.configure(new InetSocketAddress(currentClientPort), - configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS)); + if(sslEnabled) { + System.setProperty(ServerCnxnFactory.ZOOKEEPER_SERVER_CNXN_FACTORY,"org.apache.zookeeper.server.NettyServerCnxnFactory"); + String dataFileDir = !System.getProperty("test.data.files", "").isEmpty() ? System.getProperty( + "test.data.files") : configuration.get("test.data.files").replace('\\', '/').replace("c:", ""); + X509Util x509Util = new ClientX509Util(); + System.setProperty(x509Util.getSslKeystoreLocationProperty(), + dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); + System.setProperty(x509Util.getSslKeystorePasswdProperty(), KEY_STORE_TRUST_STORE_PASSWORD); + System.setProperty(x509Util.getSslTruststoreLocationProperty(), + dataFileDir + File.separator + TRUST_STORE_NAME); + System.setProperty(x509Util.getSslTruststorePasswdProperty(), KEY_STORE_TRUST_STORE_PASSWORD); + standaloneServerFactory = ServerCnxnFactory.createFactory(); + standaloneServerFactory.configure(new InetSocketAddress(currentClientPort), + configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS), true); + } else { + standaloneServerFactory = ServerCnxnFactory.createFactory(); + standaloneServerFactory.configure(new InetSocketAddress(currentClientPort), + configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, HConstants.DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS)); + } } catch (BindException e) { LOG.debug("Failed binding ZK Server to client port: " + currentClientPort, e); // We're told to use some port but it's occupied, fail @@ -252,7 +283,7 @@ public int startup(File baseDir, int numZooKeeperServers) throws IOException, In // Start up this ZK server standaloneServerFactory.startup(server); // Runs a 'stat' against the servers. - if (!waitForServerUp(currentClientPort, connectionTimeout)) { + if (!sslEnabled && !waitForServerUp(currentClientPort, connectionTimeout)) { throw new IOException("Waiting for startup of standalone server"); } @@ -292,7 +323,7 @@ private void createDir(File dir) throws IOException { public void shutdown() throws IOException { // shut down all the zk servers for (int i = 0; i < standaloneServerFactoryList.size(); i++) { - NIOServerCnxnFactory standaloneServerFactory = standaloneServerFactoryList.get(i); + ServerCnxnFactory standaloneServerFactory = standaloneServerFactoryList.get(i); int clientPort = clientPortList.get(i); standaloneServerFactory.shutdown(); @@ -328,7 +359,7 @@ public int killCurrentActiveZooKeeperServer() throws IOException, InterruptedExc } // Shutdown the current active one - NIOServerCnxnFactory standaloneServerFactory = standaloneServerFactoryList.get(activeZKServerIndex); + ServerCnxnFactory standaloneServerFactory = standaloneServerFactoryList.get(activeZKServerIndex); int clientPort = clientPortList.get(activeZKServerIndex); standaloneServerFactory.shutdown(); @@ -366,7 +397,7 @@ public void killOneBackupZooKeeperServer() throws IOException, InterruptedExcept int backupZKServerIndex = activeZKServerIndex + 1; // Shutdown the current active one - NIOServerCnxnFactory standaloneServerFactory = standaloneServerFactoryList.get(backupZKServerIndex); + ServerCnxnFactory standaloneServerFactory = standaloneServerFactoryList.get(backupZKServerIndex); int clientPort = clientPortList.get(backupZKServerIndex); standaloneServerFactory.shutdown(); diff --git service/src/java/org/apache/hive/service/server/HiveServer2.java service/src/java/org/apache/hive/service/server/HiveServer2.java index fece82e266..2046082fd5 100644 --- service/src/java/org/apache/hive/service/server/HiveServer2.java +++ service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -43,14 +43,12 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.ACLProvider; import org.apache.curator.framework.api.BackgroundCallback; import org.apache.curator.framework.api.CuratorEvent; import org.apache.curator.framework.api.CuratorEventType; import org.apache.curator.framework.recipes.leader.LeaderLatch; import org.apache.curator.framework.recipes.leader.LeaderLatchListener; -import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.JvmPauseMonitor; import org.apache.hadoop.hive.common.LogUtils; @@ -1078,14 +1076,9 @@ private void maybeStartCompactorThreads(HiveConf hiveConf) throws Exception { */ static void deleteServerInstancesFromZooKeeper(String versionNumber) throws Exception { HiveConf hiveConf = new HiveConf(); - String zooKeeperEnsemble = hiveConf.getZKConfig().getQuorumServers(); - String rootNamespace = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE); - int baseSleepTime = (int) hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_BASESLEEPTIME, TimeUnit.MILLISECONDS); - int maxRetries = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_MAX_RETRIES); - CuratorFramework zooKeeperClient = - CuratorFrameworkFactory.builder().connectString(zooKeeperEnsemble) - .retryPolicy(new ExponentialBackoffRetry(baseSleepTime, maxRetries)).build(); + CuratorFramework zooKeeperClient = hiveConf.getZKConfig().getNewZookeeperClient(null); zooKeeperClient.start(); + String rootNamespace = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE); List znodePaths = zooKeeperClient.getChildren().forPath( ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + rootNamespace); diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/SSLZookeeperFactory.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/SSLZookeeperFactory.java new file mode 100644 index 0000000000..2734415b11 --- /dev/null +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/SSLZookeeperFactory.java @@ -0,0 +1,59 @@ +package org.apache.hadoop.hive.common; + +import org.apache.commons.lang3.StringUtils; +import org.apache.curator.utils.ZookeeperFactory; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.client.ZKClientConfig; +import org.apache.zookeeper.common.ClientX509Util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.text.MessageFormat; + +/** + * Factory to create Zookeeper clients with the zookeeper.client.secure enabled, allowing SSL communication with the Zookeeper server + */ +public class SSLZookeeperFactory implements ZookeeperFactory { + + private static final Logger LOG = LoggerFactory.getLogger(SSLZookeeperFactory.class); + + private boolean sslEnabled; + private String keyStoreLocation; + private String keyStorePassword; + private String trustStoreLocation; + private String trustStorePassword; + + public SSLZookeeperFactory(boolean sslEnabled, String keyStoreLocation, String keyStorePassword, + String trustStoreLocation, String trustStorePassword) { + + this.sslEnabled = sslEnabled; + this.keyStoreLocation = keyStoreLocation; + this.keyStorePassword = keyStorePassword; + this.trustStoreLocation = trustStoreLocation; + this.trustStorePassword = trustStorePassword; + if(sslEnabled) { + if(StringUtils.isEmpty(keyStoreLocation)) { + LOG.warn("Missing keystoreLocation parameter"); + } + if(StringUtils.isEmpty(trustStoreLocation)) { + LOG.warn("Missing trustStoreLocation parameter"); + } + } + } + + @Override public ZooKeeper newZooKeeper(String connectString, int sessionTimeout, Watcher watcher, boolean canBeReadOnly) throws Exception { + if(!this.sslEnabled) { + return new ZooKeeper(connectString, sessionTimeout, watcher, canBeReadOnly); + } + ZKClientConfig clientConfig = new ZKClientConfig(); + clientConfig.setProperty(ZKClientConfig.SECURE_CLIENT, "true"); + clientConfig.setProperty(ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET,"org.apache.zookeeper.ClientCnxnSocketNetty"); + ClientX509Util x509Util = new ClientX509Util(); + clientConfig.setProperty(x509Util.getSslKeystoreLocationProperty(), this.keyStoreLocation); + clientConfig.setProperty(x509Util.getSslKeystorePasswdProperty(), this.keyStorePassword); + clientConfig.setProperty(x509Util.getSslTruststoreLocationProperty(), this.trustStoreLocation); + clientConfig.setProperty(x509Util.getSslTruststorePasswdProperty(), this.trustStorePassword); + return new ZooKeeper(connectString, sessionTimeout, watcher, canBeReadOnly,clientConfig); + } + } \ No newline at end of file diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java index 99f7c97877..97dfac0714 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java @@ -23,13 +23,20 @@ import java.util.ArrayList; import java.util.concurrent.TimeUnit; import java.util.List; + +import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.api.ACLProvider; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.nodes.PersistentEphemeralNode; import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.utils.ZookeeperFactory; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.client.ZKClientConfig; +import org.apache.zookeeper.common.ClientX509Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,16 +65,29 @@ private String quorum = null; private String clientPort = null; private String rootNamespace = null; - private boolean deregisteredWithZooKeeper = false; // Set to true only when deregistration happens + private int connectionTimeout; private int sessionTimeout; private int baseSleepTime; private int maxRetries; + private boolean sslEnabled; + private String keyStoreLocation; + private String keyStorePassword; + private String trustStoreLocation; + private String trustStorePassword; + private SSLZookeeperFactory sslZookeeperFactory; private CuratorFramework zooKeeperClient; + private boolean deregisteredWithZooKeeper = false; // Set to true only when deregistration happens private PersistentEphemeralNode znode; - public ZooKeeperHiveHelper(String quorum, String clientPort, String rootNamespace, - int sessionTimeout, int baseSleepTime, int maxRetries) { + public ZooKeeperHiveHelper(String quorum, String clientPort, String rootNamespace,int connectionTimeout, int sessionTimeout, + int baseSleepTime, int maxRetries) { + this(quorum, clientPort, rootNamespace,connectionTimeout, sessionTimeout, baseSleepTime, maxRetries, false, "", "", "", ""); + } + + public ZooKeeperHiveHelper(String quorum, String clientPort, String rootNamespace,int connectionTimeout, int sessionTimeout, + int baseSleepTime, int maxRetries, boolean sslEnabled, String keyStoreLocation, String keyStorePassword, + String trustStoreLocation, String trustStorePassword) { // Get the ensemble server addresses in the format host1:port1, host2:port2, ... . Append // the configured port to hostname if the hostname doesn't contain a port. String[] hosts = quorum.split(","); @@ -87,11 +107,19 @@ public ZooKeeperHiveHelper(String quorum, String clientPort, String rootNamespac this.quorum = quorumServers.toString(); this.clientPort = clientPort; this.rootNamespace = rootNamespace; + this.connectionTimeout = connectionTimeout; this.sessionTimeout = sessionTimeout; this.baseSleepTime = baseSleepTime; this.maxRetries = maxRetries; + this.sslEnabled = sslEnabled; + this.keyStoreLocation = keyStoreLocation; + this.keyStorePassword = keyStorePassword; + this.trustStoreLocation = trustStoreLocation; + this.trustStorePassword = trustStorePassword; + this.sslZookeeperFactory = new SSLZookeeperFactory(sslEnabled,keyStoreLocation,keyStorePassword,trustStoreLocation,trustStorePassword); } + /** * Get the ensemble server addresses. The format is: host1:port, host2:port.. **/ @@ -147,17 +175,7 @@ public void addServerInstanceToZooKeeper(String znodePathPrefix, String znodeDat public CuratorFramework startZookeeperClient(ACLProvider zooKeeperAclProvider, boolean addParentNode) throws Exception { - String zooKeeperEnsemble = getQuorumServers(); - // Create a CuratorFramework instance to be used as the ZooKeeper client. - // Use the zooKeeperAclProvider, when specified, to create appropriate ACLs. - CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() - .connectString(zooKeeperEnsemble) - .sessionTimeoutMs(sessionTimeout) - .retryPolicy(new ExponentialBackoffRetry(baseSleepTime, maxRetries)); - if (zooKeeperAclProvider != null) { - builder = builder.aclProvider(zooKeeperAclProvider); - } - CuratorFramework zkClient = builder.build(); + CuratorFramework zkClient = getNewZookeeperClient(zooKeeperAclProvider); zkClient.start(); // Create the parent znodes recursively; ignore if the parent already exists. @@ -178,6 +196,26 @@ public CuratorFramework startZookeeperClient(ACLProvider zooKeeperAclProvider, return zkClient; } + public CuratorFramework getNewZookeeperClient(ACLProvider zooKeeperAclProvider) { + String zooKeeperEnsemble = getQuorumServers(); + LOG.info("Creating curator client with connectString: {} sessionTimeoutMs: {} connectionTimeoutMs: {}" + + " exponentialBackoff - sleepTime: {} maxRetries: {} sslEnabled: {}", zooKeeperEnsemble, sessionTimeout, + connectionTimeout, baseSleepTime, maxRetries, sslEnabled); + // Create a CuratorFramework instance to be used as the ZooKeeper client. + // Use the zooKeeperAclProvider, when specified, to create appropriate ACLs. + CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() + .connectString(zooKeeperEnsemble) + .connectionTimeoutMs(connectionTimeout) + .sessionTimeoutMs(sessionTimeout) + .retryPolicy(new ExponentialBackoffRetry(baseSleepTime, maxRetries)) + .zookeeperFactory(this.sslZookeeperFactory); + if (zooKeeperAclProvider != null) { + builder = builder.aclProvider(zooKeeperAclProvider); + } + + return builder.build(); + } + public void removeServerInstanceFromZooKeeper() throws Exception { setDeregisteredWithZooKeeper(true); diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index fc6a2fd43a..6d38edb92f 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1072,32 +1072,47 @@ public static ConfVars getMetaConf(String name) { "If ZooKeeper is configured for Kerberos authentication. This could be useful when cluster\n" + "is kerberized, but Zookeeper is not."), THRIFT_ZOOKEEPER_CLIENT_PORT("metastore.zookeeper.client.port", - "hive.metastore.zookeeper.client.port", "2181", + "hive.zookeeper.client.port", "2181", "The port of ZooKeeper servers to talk to.\n" + "If the list of Zookeeper servers specified in hive.metastore.thrift.uris" + " does not contain port numbers, this value is used."), THRIFT_ZOOKEEPER_SESSION_TIMEOUT("metastore.zookeeper.session.timeout", - "hive.metastore.zookeeper.session.timeout", 120000L, TimeUnit.MILLISECONDS, + "hive.zookeeper.session.timeout", 120000L, TimeUnit.MILLISECONDS, new TimeValidator(TimeUnit.MILLISECONDS), "ZooKeeper client's session timeout (in milliseconds). The client is disconnected\n" + "if a heartbeat is not sent in the timeout."), THRIFT_ZOOKEEPER_CONNECTION_TIMEOUT("metastore.zookeeper.connection.timeout", - "hive.metastore.zookeeper.connection.timeout", 15L, TimeUnit.SECONDS, + "hive.zookeeper.connection.timeout", 15L, TimeUnit.SECONDS, new TimeValidator(TimeUnit.SECONDS), "ZooKeeper client's connection timeout in seconds. " + "Connection timeout * hive.metastore.zookeeper.connection.max.retries\n" + "with exponential backoff is when curator client deems connection is lost to zookeeper."), THRIFT_ZOOKEEPER_NAMESPACE("metastore.zookeeper.namespace", - "hive.metastore.zookeeper.namespace", "hive_metastore", + "hive.zookeeper.namespace", "hive_metastore", "The parent node under which all ZooKeeper nodes for metastores are created."), THRIFT_ZOOKEEPER_CONNECTION_MAX_RETRIES("metastore.zookeeper.connection.max.retries", - "hive.metastore.zookeeper.connection.max.retries", 3, + "hive.zookeeper.connection.max.retries", 3, "Max number of times to retry when connecting to the ZooKeeper server."), THRIFT_ZOOKEEPER_CONNECTION_BASESLEEPTIME("metastore.zookeeper.connection.basesleeptime", - "hive.metastore.zookeeper.connection.basesleeptime", 1000L, TimeUnit.MILLISECONDS, + "hive.zookeeper.connection.basesleeptime", 1000L, TimeUnit.MILLISECONDS, new TimeValidator(TimeUnit.MILLISECONDS), "Initial amount of time (in milliseconds) to wait between retries\n" + "when connecting to the ZooKeeper server when using ExponentialBackoffRetry policy."), + THRIFT_ZOOKEEPER_SSL_ENABLE("metastore.zookeeper.ssl.client.enable","hive.zookeeper.ssl.client.enable", "false", + "Set client to use TLS when connecting to ZooKeeper. An explicit value overrides any value set via the " + + "zookeeper.client.secure system property (note the different name). Defaults to false if neither is set."), + THRIFT_ZOOKEEPER_SSL_KEYSTORE_LOCATION("metastore.zookeeper.ssl.keystore.location","hive.zookeeper.ssl.keystore.location","", + "Keystore location when using a client-side certificate with TLS connectivity to ZooKeeper. " + + "Overrides any explicit value set via the zookeeper.ssl.keyStore.location system property (note the camelCase)."), + THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD("metastore.zookeeper.ssl.keystore.password","hive.zookeeper.ssl.keystore.password","", + "Keystore password when using a client-side certificate with TLS connectivity to ZooKeeper." + + "Overrides any explicit value set via the zookeeper.ssl.keyStore.password system property (note the camelCase)."), + THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION("metastore.zookeeper.ssl.truststore.location","hive.zookeeper.ssl.truststore.location","", + "Truststore location when using a client-side certificate with TLS connectivity to ZooKeeper. " + + "Overrides any explicit value set via the zookeeper.ssl.trustStore.location system property (note the camelCase)."), + THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD("metastore.zookeeper.ssl.truststore.password","hive.zookeeper.ssl.truststore.password","", + "Truststore password when using a client-side certificate with TLS connectivity to ZooKeeper." + + "Overrides any explicit value set via the zookeeper.ssl.trustStore.password system property (note the camelCase)."), THRIFT_URI_SELECTION("metastore.thrift.uri.selection", "hive.metastore.uri.selection", "RANDOM", new StringSetValidator("RANDOM", "SEQUENTIAL"), "Determines the selection mechanism used by metastore client to connect to remote " + @@ -2021,11 +2036,17 @@ public static ZooKeeperHiveHelper getZKConfig(Configuration conf) { return new ZooKeeperHiveHelper(MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS), MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_CLIENT_PORT), MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_NAMESPACE), + (int) getTimeVar(conf, ConfVars.THRIFT_ZOOKEEPER_CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS), (int) MetastoreConf.getTimeVar(conf, ConfVars.THRIFT_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS), (int) MetastoreConf.getTimeVar(conf, ConfVars.THRIFT_ZOOKEEPER_CONNECTION_BASESLEEPTIME, TimeUnit.MILLISECONDS), - MetastoreConf.getIntVar(conf, ConfVars.THRIFT_ZOOKEEPER_CONNECTION_MAX_RETRIES)); + MetastoreConf.getIntVar(conf, ConfVars.THRIFT_ZOOKEEPER_CONNECTION_MAX_RETRIES), + MetastoreConf.getBoolVar(conf,ConfVars.THRIFT_ZOOKEEPER_SSL_ENABLE), + MetastoreConf.getVar(conf,ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_LOCATION), + MetastoreConf.getVar(conf,ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD), + MetastoreConf.getVar(conf,ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION), + MetastoreConf.getVar(conf,ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD)); } /** diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java index b35dc7ce4b..239bff6dc9 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java @@ -46,6 +46,16 @@ public static final String DELEGATION_TOKEN_STORE_ZK_ACL = "hive.cluster.delegation.token.store.zookeeper.acl"; public static final String DELEGATION_TOKEN_STORE_ZK_ZNODE_DEFAULT = "/hivedelegation"; + public static final String DELEGATION_TOKEN_STORE_ZK_SSL_ENABLE = + "hive.cluster.delegation.token.store.zookeeper.ssl.client.enable"; + public static final String DELEGATION_TOKEN_STORE_ZK_KEYSTORE_LOCATION = + "hive.cluster.delegation.token.store.zookeeper.keystore.location"; + public static final String DELEGATION_TOKEN_STORE_ZK_KEYSTORE_PASSWORD = + "hive.cluster.delegation.token.store.zookeeper.keystore.password"; + public static final String DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_LOCATION = + "hive.cluster.delegation.token.store.zookeeper.truststore.location"; + public static final String DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_PASSWORD = + "hive.cluster.delegation.token.store.zookeeper.truststore.password"; public MetastoreDelegationTokenManager() { } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java index af52fcc5f6..a8aaee47fe 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java @@ -31,6 +31,7 @@ import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.SSLZookeeperFactory; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -57,12 +58,22 @@ protected static final String ZK_SEQ_FORMAT = "%010d"; private static final String NODE_KEYS = "/keys"; private static final String NODE_TOKENS = "/tokens"; + private static final String WHEN_ZK_DSTORE_MSG = "when zookeeper based delegation token storage is enabled" + + "(hive.cluster.delegation.token.store.class=" + ZooKeeperTokenStore.class.getName() + ")"; private String rootNode = ""; private volatile CuratorFramework zkSession; private String zkConnectString; private int connectTimeoutMillis; + private boolean sslEnabled; + private String keyStoreLocation; + private String keyStorePassword; + private String trustStoreLocation; + private String trustStorePassword; + private List newNodeAcl; + private Configuration conf; + private HadoopThriftAuthBridge.Server.ServerMode serverMode; /** * ACLProvider permissions will be used in case parent dirs need to be created @@ -110,12 +121,7 @@ private boolean isKerberosEnabled(Configuration conf) { } } - private final String WHEN_ZK_DSTORE_MSG = "when zookeeper based delegation token storage is enabled" - + "(hive.cluster.delegation.token.store.class=" + ZooKeeperTokenStore.class.getName() + ")"; - - private Configuration conf; - private HadoopThriftAuthBridge.Server.ServerMode serverMode; /** * Default constructor for dynamic instantiation w/ Configurable @@ -124,14 +130,15 @@ private boolean isKerberosEnabled(Configuration conf) { protected ZooKeeperTokenStore() { } - private CuratorFramework getSession() { + public CuratorFramework getSession() { if (zkSession == null || zkSession.getState() == CuratorFrameworkState.STOPPED) { synchronized (this) { if (zkSession == null || zkSession.getState() == CuratorFrameworkState.STOPPED) { zkSession = CuratorFrameworkFactory.builder().connectString(zkConnectString) .connectionTimeoutMs(connectTimeoutMillis).aclProvider(aclDefaultProvider) - .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build(); + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .zookeeperFactory(new SSLZookeeperFactory(sslEnabled,keyStoreLocation,keyStorePassword,trustStoreLocation,trustStorePassword)).build(); zkSession.start(); } } @@ -478,10 +485,14 @@ public void init(Object hmsHandler, HadoopThriftAuthBridge.Server.ServerMode sMo + WHEN_ZK_DSTORE_MSG); } } - connectTimeoutMillis = - conf.getInt( - MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS, - CuratorFrameworkFactory.builder().getConnectionTimeoutMs()); + connectTimeoutMillis = conf.getInt(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_CONNECT_TIMEOUTMILLIS, + CuratorFrameworkFactory.builder().getConnectionTimeoutMs()); + + sslEnabled = conf.getBoolean(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_SSL_ENABLE,false); + keyStoreLocation = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_LOCATION,""); + keyStorePassword = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_PASSWORD,""); + trustStoreLocation = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_LOCATION,""); + trustStorePassword = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_PASSWORD,""); String aclStr = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_ACL, null); this.newNodeAcl = StringUtils.isNotBlank(aclStr)? parseACLs(aclStr) : getDefaultAcl(conf);