diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 97cffff256..ff3b0263bb 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2582,7 +2582,13 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "this is set to false, however unless MAPREDUCE-7086 fix is present, queries that\n" + "read MM tables with original files will fail. The default in Hive 3.0 is false."), - // Zookeeper related configs + // Zookeeper related configs + HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION("hive.security.zookeeper.authentication", + "default", new StringSet("default", "simple"), + "Whether authentication type configured for Zookeeper is different from the default cluster wide\n" + + "set under `hadoop.security.authentication`. This could be useful in case when cluster\n" + + "is kerberized, but Zookeeper is not."), + HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", "", "List of ZooKeeper servers to talk to. This is needed for: \n" + "1. Read/write locks - when hive.lock.manager is set to \n" + diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java index e56ae114fe..5751b8ed93 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java +++ b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java @@ -160,7 +160,7 @@ public ZkRegistryBase(String instanceName, Configuration conf, String rootNs, St this.stateChangeListeners = new HashSet<>(); this.pathToInstanceCache = new ConcurrentHashMap<>(); this.nodeToInstanceCache = new ConcurrentHashMap<>(); - final String namespace = getRootNamespace(rootNs, nsPrefix); + final String namespace = getRootNamespace(conf, rootNs, nsPrefix); ACLProvider aclProvider; // get acl provider for most outer path that is non-null if (userPathPrefix == null) { @@ -180,8 +180,9 @@ public ZkRegistryBase(String instanceName, Configuration conf, String rootNs, St this.zooKeeperClient.getConnectionStateListenable().addListener(new ZkConnectionStateListener()); } - public static String getRootNamespace(String userProvidedNamespace, String defaultNamespacePrefix) { - final boolean isSecure = UserGroupInformation.isSecurityEnabled(); + public static String getRootNamespace(Configuration conf, String userProvidedNamespace, + String defaultNamespacePrefix) { + final boolean isSecure = ZookeeperUtils.isKerberosEnabled(conf); String rootNs = userProvidedNamespace; if (rootNs == null) { rootNs = defaultNamespacePrefix + (isSecure ? SASL_NAMESPACE : UNSECURE_NAMESPACE); @@ -190,7 +191,7 @@ public static String getRootNamespace(String userProvidedNamespace, String defau } private ACLProvider getACLProviderForZKPath(String zkPath) { - final boolean isSecure = UserGroupInformation.isSecurityEnabled(); + final boolean isSecure = ZookeeperUtils.isKerberosEnabled(conf); return new ACLProvider() { @Override public List getDefaultAcl() { @@ -366,7 +367,9 @@ final void initializeWithoutRegisteringInternal() throws IOException { } private void checkAndSetAcls() throws Exception { - if (!UserGroupInformation.isSecurityEnabled()) return; + if (!ZookeeperUtils.isKerberosEnabled(conf)) { + return; + } // We are trying to check ACLs on the "workers" directory, which noone except us should be // able to write to. Higher-level directories shouldn't matter - we don't read them. String pathToCheck = workersPath; diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java index 454d503454..2b1204194e 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java +++ b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java @@ -13,6 +13,8 @@ */ package org.apache.hadoop.hive.registry.impl; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,7 +34,7 @@ public static String setupZookeeperAuth(Configuration conf, String saslLoginContextName, String zkPrincipal, String zkKeytab) throws IOException { // If the login context name is not set, we are in the client and don't need auth. - if (UserGroupInformation.isSecurityEnabled() && saslLoginContextName != null) { + if (isKerberosEnabled(conf) && saslLoginContextName != null) { LOG.info("UGI security is enabled. Setting up ZK auth."); if (zkPrincipal == null || zkPrincipal.isEmpty()) { @@ -52,6 +54,18 @@ public static String setupZookeeperAuth(Configuration conf, String saslLoginCont } } + /** + * Check if Kerberos is enabled + */ + public static boolean isKerberosEnabled(Configuration conf) { + try { + return UserGroupInformation.getLoginUser().isFromKeytab() && !AuthenticationMethod.SIMPLE.name().equalsIgnoreCase( + HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION)); + } catch (IOException e) { + return false; + } + } + /** * Dynamically sets up the JAAS configuration that uses kerberos * diff --git a/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java b/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java new file mode 100644 index 0000000000..887c75d84e --- /dev/null +++ b/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.registry.impl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * ZookeeperUtils test suite. + */ +public class TestZookeeperUtils { + + private Configuration conf; + private UserGroupInformation ugi; + + @Before + public void setup() { + conf = new Configuration(); + ugi = Mockito.mock(UserGroupInformation.class); + UserGroupInformation.setLoginUser(ugi); + } + + @Test + public void testHadoopKerberosZookeeperDefault() { + Mockito.when(ugi.isFromKeytab()).thenReturn(true); + Assert.assertTrue(ZookeeperUtils.isKerberosEnabled(conf)); + } + + @Test + public void testHadoopKerberosZookeeperSimple(){ + Mockito.when(ugi.isFromKeytab()).thenReturn(true); + conf.set(HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION.varname, + AuthenticationMethod.SIMPLE.name()); + + Assert.assertFalse(ZookeeperUtils.isKerberosEnabled(conf)); + } + + @Test + public void testHadoopSimpleZookeeperDefault(){ + Mockito.when(ugi.isFromKeytab()).thenReturn(false); + conf.set(HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION.varname, + AuthenticationMethod.SIMPLE.name()); + + Assert.assertFalse(ZookeeperUtils.isKerberosEnabled(conf)); + } +} diff --git a/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java b/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java index f87b610ee2..122742e021 100644 --- a/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java +++ b/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java @@ -40,7 +40,7 @@ public static synchronized HS2ActivePassiveHARegistry getClient(Configuration co String namespace = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_REGISTRY_NAMESPACE); Preconditions.checkArgument(!StringUtils.isBlank(namespace), HiveConf.ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_REGISTRY_NAMESPACE.varname + " cannot be null or empty"); - String nsKey = ZkRegistryBase.getRootNamespace(null, namespace + "-"); + String nsKey = ZkRegistryBase.getRootNamespace(conf, null, namespace + "-"); HS2ActivePassiveHARegistry registry = hs2Registries.get(nsKey); if (registry == null) { registry = HS2ActivePassiveHARegistry.create(conf, true); diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index 5d81668441..17570f73bc 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -86,9 +86,9 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.txn.compactor.CompactorThread; import org.apache.hadoop.hive.ql.txn.compactor.Worker; +import org.apache.hadoop.hive.registry.impl.ZookeeperUtils; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.common.util.HiveVersionInfo; import org.apache.hive.common.util.ShutdownHookManager; @@ -278,6 +278,7 @@ public void run() { } wmQueue = hiveConf.get(ConfVars.HIVE_SERVER2_TEZ_INTERACTIVE_QUEUE.varname, "").trim(); + this.zooKeeperAclProvider = getACLProvider(hiveConf); this.serviceDiscovery = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY); this.activePassiveHA = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_ENABLE); @@ -453,29 +454,34 @@ public static boolean isKerberosAuthMode(Configuration hiveConf) { /** * ACLProvider for providing appropriate ACLs to CuratorFrameworkFactory */ - private final ACLProvider zooKeeperAclProvider = new ACLProvider() { + private ACLProvider zooKeeperAclProvider; - @Override - public List getDefaultAcl() { - List nodeAcls = new ArrayList(); - if (UserGroupInformation.isSecurityEnabled()) { - // Read all to the world - nodeAcls.addAll(Ids.READ_ACL_UNSAFE); - // Create/Delete/Write/Admin to the authenticated user - nodeAcls.add(new ACL(Perms.ALL, Ids.AUTH_IDS)); - } else { - // ACLs for znodes on a non-kerberized cluster - // Create/Read/Delete/Write/Admin to the world - nodeAcls.addAll(Ids.OPEN_ACL_UNSAFE); + private ACLProvider getACLProvider(HiveConf hiveConf) { + final boolean isSecure = ZookeeperUtils.isKerberosEnabled(hiveConf); + + return new ACLProvider() { + @Override + public List getDefaultAcl() { + List nodeAcls = new ArrayList(); + if (isSecure) { + // Read all to the world + nodeAcls.addAll(Ids.READ_ACL_UNSAFE); + // Create/Delete/Write/Admin to the authenticated user + nodeAcls.add(new ACL(Perms.ALL, Ids.AUTH_IDS)); + } else { + // ACLs for znodes on a non-kerberized cluster + // Create/Read/Delete/Write/Admin to the world + nodeAcls.addAll(Ids.OPEN_ACL_UNSAFE); + } + return nodeAcls; } - return nodeAcls; - } - @Override - public List getAclForPath(String path) { - return getDefaultAcl(); - } - }; + @Override + public List getAclForPath(String path) { + return getDefaultAcl(); + } + }; + } /** * Add conf keys, values that HiveServer2 will publish to ZooKeeper. @@ -522,7 +528,7 @@ private void addConfsToPublish(HiveConf hiveConf, Map confsToPub * @throws Exception */ private void setUpZooKeeperAuth(HiveConf hiveConf) throws Exception { - if (UserGroupInformation.isSecurityEnabled()) { + if (ZookeeperUtils.isKerberosEnabled(hiveConf)) { String principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL); if (principal.isEmpty()) { throw new IOException("HiveServer2 Kerberos principal is empty"); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java index 953c5fd1d0..ba1f1774d3 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java @@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; import org.apache.hadoop.security.token.delegation.MetastoreDelegationTokenSupport; import org.apache.zookeeper.CreateMode; @@ -111,7 +112,8 @@ private CuratorFramework getSession() { } private void setupJAASConfig(Configuration conf) throws IOException { - if (!UserGroupInformation.getLoginUser().isFromKeytab()) { + if (!UserGroupInformation.getLoginUser().isFromKeytab() || AuthenticationMethod.SIMPLE.name().equalsIgnoreCase( + getNonEmptyConfVar(conf, "hive.security.zookeeper.authentication"))) { // The process has not logged in using keytab // this should be a test mode, can't use keytab to authenticate // with zookeeper.