diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 97cffff256..ff3b0263bb 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2582,7 +2582,13 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "this is set to false, however unless MAPREDUCE-7086 fix is present, queries that\n" + "read MM tables with original files will fail. The default in Hive 3.0 is false."), - // Zookeeper related configs + // Zookeeper related configs + HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION("hive.security.zookeeper.authentication", + "default", new StringSet("default", "simple"), + "Whether authentication type configured for Zookeeper is different from the default cluster wide\n" + + "set under `hadoop.security.authentication`. This could be useful in case when cluster\n" + + "is kerberized, but Zookeeper is not."), + HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", "", "List of ZooKeeper servers to talk to. This is needed for: \n" + "1. Read/write locks - when hive.lock.manager is set to \n" + diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java index e56ae114fe..26f09e3413 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java +++ b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java @@ -160,7 +160,7 @@ public ZkRegistryBase(String instanceName, Configuration conf, String rootNs, St this.stateChangeListeners = new HashSet<>(); this.pathToInstanceCache = new ConcurrentHashMap<>(); this.nodeToInstanceCache = new ConcurrentHashMap<>(); - final String namespace = getRootNamespace(rootNs, nsPrefix); + final String namespace = getRootNamespace(conf, rootNs, nsPrefix); ACLProvider aclProvider; // get acl provider for most outer path that is non-null if (userPathPrefix == null) { @@ -180,8 +180,8 @@ public ZkRegistryBase(String instanceName, Configuration conf, String rootNs, St this.zooKeeperClient.getConnectionStateListenable().addListener(new ZkConnectionStateListener()); } - public static String getRootNamespace(String userProvidedNamespace, String defaultNamespacePrefix) { - final boolean isSecure = UserGroupInformation.isSecurityEnabled(); + public static String getRootNamespace(Configuration conf, String userProvidedNamespace, String defaultNamespacePrefix) { + final boolean isSecure = ZookeeperUtils.isSecurityEnabled(conf); String rootNs = userProvidedNamespace; if (rootNs == null) { rootNs = defaultNamespacePrefix + (isSecure ? SASL_NAMESPACE : UNSECURE_NAMESPACE); @@ -190,7 +190,7 @@ public static String getRootNamespace(String userProvidedNamespace, String defau } private ACLProvider getACLProviderForZKPath(String zkPath) { - final boolean isSecure = UserGroupInformation.isSecurityEnabled(); + final boolean isSecure = ZookeeperUtils.isSecurityEnabled(conf); return new ACLProvider() { @Override public List getDefaultAcl() { @@ -366,7 +366,7 @@ final void initializeWithoutRegisteringInternal() throws IOException { } private void checkAndSetAcls() throws Exception { - if (!UserGroupInformation.isSecurityEnabled()) return; + if (!ZookeeperUtils.isSecurityEnabled(conf)) return; // We are trying to check ACLs on the "workers" directory, which noone except us should be // able to write to. Higher-level directories shouldn't matter - we don't read them. String pathToCheck = workersPath; diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java index 454d503454..7d307500e8 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java +++ b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java @@ -13,6 +13,8 @@ */ package org.apache.hadoop.hive.registry.impl; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,7 +34,7 @@ public static String setupZookeeperAuth(Configuration conf, String saslLoginContextName, String zkPrincipal, String zkKeytab) throws IOException { // If the login context name is not set, we are in the client and don't need auth. - if (UserGroupInformation.isSecurityEnabled() && saslLoginContextName != null) { + if (isSecurityEnabled(conf) && saslLoginContextName != null) { LOG.info("UGI security is enabled. Setting up ZK auth."); if (zkPrincipal == null || zkPrincipal.isEmpty()) { @@ -52,6 +54,11 @@ public static String setupZookeeperAuth(Configuration conf, String saslLoginCont } } + public static boolean isSecurityEnabled(Configuration conf) { + return UserGroupInformation.isSecurityEnabled() && !AuthenticationMethod.SIMPLE.name().equalsIgnoreCase( + HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION)); + } + /** * Dynamically sets up the JAAS configuration that uses kerberos * diff --git a/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java b/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java new file mode 100644 index 0000000000..d52ff5f76d --- /dev/null +++ b/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java @@ -0,0 +1,51 @@ +package org.apache.hadoop.hive.registry.impl; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestZookeeperUtils { + + private Configuration conf; + + @Before + public void setup() { + conf = new Configuration(); + + System.setProperty("java.security.krb5.realm", "hive"); + System.setProperty("java.security.krb5.kdc", "fake"); + } + + @Test + public void testHadoopKerberosZookeeperDefault(){ + conf.set(HADOOP_SECURITY_AUTHENTICATION, AuthenticationMethod.KERBEROS.name()); + UserGroupInformation.setConfiguration(conf); + + Assert.assertTrue(ZookeeperUtils.isSecurityEnabled(conf)); + } + + @Test + public void testHadoopKerberosZookeeperSimple(){ + conf.set(HADOOP_SECURITY_AUTHENTICATION, AuthenticationMethod.KERBEROS.name()); + conf.set(HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION.varname, AuthenticationMethod.SIMPLE.name()); + UserGroupInformation.setConfiguration(conf); + + Assert.assertFalse(ZookeeperUtils.isSecurityEnabled(conf)); + } + + @Test + public void testHadoopSimpleZookeeperDefault(){ + conf.set(HADOOP_SECURITY_AUTHENTICATION, AuthenticationMethod.SIMPLE.name()); + conf.set(HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION.varname, AuthenticationMethod.SIMPLE.name()); + UserGroupInformation.setConfiguration(conf); + + Assert.assertFalse(ZookeeperUtils.isSecurityEnabled(conf)); + } + +} diff --git a/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java b/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java index f87b610ee2..122742e021 100644 --- a/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java +++ b/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java @@ -40,7 +40,7 @@ public static synchronized HS2ActivePassiveHARegistry getClient(Configuration co String namespace = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_REGISTRY_NAMESPACE); Preconditions.checkArgument(!StringUtils.isBlank(namespace), HiveConf.ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_REGISTRY_NAMESPACE.varname + " cannot be null or empty"); - String nsKey = ZkRegistryBase.getRootNamespace(null, namespace + "-"); + String nsKey = ZkRegistryBase.getRootNamespace(conf, null, namespace + "-"); HS2ActivePassiveHARegistry registry = hs2Registries.get(nsKey); if (registry == null) { registry = HS2ActivePassiveHARegistry.create(conf, true); diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index 5d81668441..13483bc748 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -86,6 +86,7 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.txn.compactor.CompactorThread; import org.apache.hadoop.hive.ql.txn.compactor.Worker; +import org.apache.hadoop.hive.registry.impl.ZookeeperUtils; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; @@ -278,6 +279,7 @@ public void run() { } wmQueue = hiveConf.get(ConfVars.HIVE_SERVER2_TEZ_INTERACTIVE_QUEUE.varname, "").trim(); + this.zooKeeperAclProvider = getACLProvider(hiveConf); this.serviceDiscovery = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY); this.activePassiveHA = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_ENABLE); @@ -453,29 +455,34 @@ public static boolean isKerberosAuthMode(Configuration hiveConf) { /** * ACLProvider for providing appropriate ACLs to CuratorFrameworkFactory */ - private final ACLProvider zooKeeperAclProvider = new ACLProvider() { + private ACLProvider zooKeeperAclProvider; - @Override - public List getDefaultAcl() { - List nodeAcls = new ArrayList(); - if (UserGroupInformation.isSecurityEnabled()) { - // Read all to the world - nodeAcls.addAll(Ids.READ_ACL_UNSAFE); - // Create/Delete/Write/Admin to the authenticated user - nodeAcls.add(new ACL(Perms.ALL, Ids.AUTH_IDS)); - } else { - // ACLs for znodes on a non-kerberized cluster - // Create/Read/Delete/Write/Admin to the world - nodeAcls.addAll(Ids.OPEN_ACL_UNSAFE); + private ACLProvider getACLProvider(HiveConf hiveConf) { + final boolean isSecure = ZookeeperUtils.isSecurityEnabled(hiveConf); + + return new ACLProvider() { + @Override + public List getDefaultAcl() { + List nodeAcls = new ArrayList(); + if (isSecure) { + // Read all to the world + nodeAcls.addAll(Ids.READ_ACL_UNSAFE); + // Create/Delete/Write/Admin to the authenticated user + nodeAcls.add(new ACL(Perms.ALL, Ids.AUTH_IDS)); + } else { + // ACLs for znodes on a non-kerberized cluster + // Create/Read/Delete/Write/Admin to the world + nodeAcls.addAll(Ids.OPEN_ACL_UNSAFE); + } + return nodeAcls; } - return nodeAcls; - } - @Override - public List getAclForPath(String path) { - return getDefaultAcl(); - } - }; + @Override + public List getAclForPath(String path) { + return getDefaultAcl(); + } + }; + } /** * Add conf keys, values that HiveServer2 will publish to ZooKeeper. @@ -522,7 +529,7 @@ private void addConfsToPublish(HiveConf hiveConf, Map confsToPub * @throws Exception */ private void setUpZooKeeperAuth(HiveConf hiveConf) throws Exception { - if (UserGroupInformation.isSecurityEnabled()) { + if (ZookeeperUtils.isSecurityEnabled(hiveConf)) { String principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL); if (principal.isEmpty()) { throw new IOException("HiveServer2 Kerberos principal is empty");