commit 9fcbb2962d74c9d5a1f7ca0e5578fbd710135e93 Author: Vihang Karajgaonkar Date: Fri Sep 1 19:14:35 2017 -0700 HIVE-17368 : DBTokenStore fails to connect in Kerberos enabled remote HMS environment diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java index bbec37e..c02879d 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java @@ -51,6 +51,7 @@ public static String HIVE_TEST_USER_2 = "user2"; public static String HIVE_TEST_SUPER_USER = "superuser"; public static String AUTHENTICATION_TYPE = "KERBEROS"; + private static final String HIVE_METASTORE_SERVICE_PRINCIPAL = "hive"; private final MiniKdc miniKdc; private final File workDir; @@ -204,6 +205,39 @@ public static MiniHS2 getMiniHS2WithKerbWithRemoteHMS(MiniHiveKdc miniHiveKdc, H return getMiniHS2WithKerbWithRemoteHMS(miniHiveKdc, hiveConf, AUTHENTICATION_TYPE); } + public static MiniHS2 getMiniHS2WithKerbWithRemoteHMSWithKerb(MiniHiveKdc miniHiveKdc, + HiveConf hiveConf) throws Exception { + return getMiniHS2WithKerbWithRemoteHMSWithKerb(miniHiveKdc, hiveConf, AUTHENTICATION_TYPE); + } + + /** + * Create a MiniHS2 with the hive service principal and keytab in MiniHiveKdc. It uses remote HMS + * and can support a different Sasl authType. It creates a metastore service principal and keytab + * which can be used for secure HMS + * @param miniHiveKdc + * @param hiveConf + * @param authenticationType + * @return new MiniHS2 instance + * @throws Exception + */ + private static MiniHS2 getMiniHS2WithKerbWithRemoteHMSWithKerb(MiniHiveKdc miniHiveKdc, + HiveConf hiveConf, String authenticationType) throws Exception { + String hivePrincipal = + miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL); + String hiveKeytab = miniHiveKdc.getKeyTabFile( + miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL)); + + String hiveMetastorePrincipal = + miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_METASTORE_SERVICE_PRINCIPAL); + String hiveMetastoreKeytab = miniHiveKdc.getKeyTabFile( + miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_METASTORE_SERVICE_PRINCIPAL)); + + return new MiniHS2.Builder().withConf(hiveConf) + .withSecureRemoteMetastore(hiveMetastorePrincipal, hiveMetastoreKeytab). + withMiniKdc(hivePrincipal, hiveKeytab).withAuthenticationType(authenticationType) + .build(); + } + /** * Create a MiniHS2 with the hive service principal and keytab in MiniHiveKdc. It uses remote HMS * and can support a different Sasl authType diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java index d690aaa..fc1dc49 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java @@ -34,7 +34,11 @@ public static void beforeTest() throws Exception { HiveConf hiveConf = new HiveConf(); hiveConf.setVar(ConfVars.METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS, "org.apache.hadoop.hive.thrift.DBTokenStore"); miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf); - miniHS2 = MiniHiveKdc.getMiniHS2WithKerbWithRemoteHMS(miniHiveKdc, hiveConf); + miniHS2 = MiniHiveKdc.getMiniHS2WithKerbWithRemoteHMSWithKerb(miniHiveKdc, hiveConf); miniHS2.start(confOverlay); + String metastorePrincipal = miniHS2.getConfProperty(ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname); + String hs2Principal = miniHS2.getConfProperty(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL.varname); + String hs2KeyTab = miniHS2.getConfProperty(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB.varname); + System.out.println("HS2 principal : " + hs2Principal + " HS2 keytab : " + hs2KeyTab + " Metastore principal : " + metastorePrincipal); } } \ No newline at end of file diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java index 36a9ea8..7e5005c 100644 --- a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java +++ b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java @@ -169,7 +169,7 @@ public void testDelegationTokenSharedStore() throws Exception { tokenManager.startThreads(); tokenManager.stopThreads(); - String tokenStrForm = tokenManager.getDelegationToken(clientUgi.getShortUserName()); + String tokenStrForm = tokenManager.getDelegationToken(clientUgi.getShortUserName(), clientUgi.getShortUserName()); Token t= new Token(); t.decodeFromUrlString(tokenStrForm); diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 71f9640..ebc4c10 100644 --- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -72,6 +72,7 @@ private final String serverPrincipal; private final boolean isMetastoreRemote; private final boolean cleanupLocalDirOnStartup; + private final boolean isMetastoreSecure; private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; public enum MiniClusterType { @@ -93,6 +94,9 @@ private String authType = "KERBEROS"; private boolean isHA = false; private boolean cleanupLocalDirOnStartup = true; + private boolean isMetastoreSecure; + private String metastoreServerPrincipal; + private String metastoreServerKeyTab; public Builder() { } @@ -119,6 +123,14 @@ public Builder withRemoteMetastore() { return this; } + public Builder withSecureRemoteMetastore(String metastoreServerPrincipal, String metastoreServerKeyTab) { + this.isMetastoreRemote = true; + this.isMetastoreSecure = true; + this.metastoreServerPrincipal = metastoreServerPrincipal; + this.metastoreServerKeyTab = metastoreServerKeyTab; + return this; + } + public Builder withConf(HiveConf hiveConf) { this.hiveConf = hiveConf; return this; @@ -153,7 +165,8 @@ public MiniHS2 build() throws Exception { hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_BINARY_MODE); } return new MiniHS2(hiveConf, miniClusterType, useMiniKdc, serverPrincipal, serverKeytab, - isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup); + isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup, + isMetastoreSecure, metastoreServerPrincipal, metastoreServerKeyTab); } } @@ -191,7 +204,10 @@ public boolean isUseMiniKdc() { private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useMiniKdc, String serverPrincipal, String serverKeytab, boolean isMetastoreRemote, - boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup) throws Exception { + boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup, + boolean isMetastoreSecure, + String metastoreServerPrincipal, + String metastoreKeyTab) throws Exception { // Always use localhost for hostname as some tests like SSL CN validation ones // are tied to localhost being present in the certificate name super( @@ -208,6 +224,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM this.useMiniKdc = useMiniKdc; this.serverPrincipal = serverPrincipal; this.isMetastoreRemote = isMetastoreRemote; + this.isMetastoreSecure = isMetastoreSecure; this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup; baseDir = getBaseDir(); localFS = FileSystem.getLocal(hiveConf); @@ -261,9 +278,15 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB, serverKeytab); hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, authType); } - String metaStoreURL = - "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator - + "test_metastore;create=true"; + + String metaStoreURL = "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator + + "test_metastore;create=true"; + + if (isMetastoreSecure) { + hiveConf.setVar(ConfVars.METASTORE_KERBEROS_PRINCIPAL, metastoreServerPrincipal); + hiveConf.setVar(ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, metastoreKeyTab); + hiveConf.setBoolVar(ConfVars.METASTORE_USE_THRIFT_SASL, true); + } fs.mkdirs(baseFsDir); Path wareHouseDir = new Path(baseFsDir, "warehouse"); @@ -301,10 +324,11 @@ public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType) throws Exception this(hiveConf, clusterType, false); } - public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, - boolean usePortsFromConf) throws Exception { - this(hiveConf, clusterType, false, null, null, false, usePortsFromConf, - "KERBEROS", false, true); + public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsFromConf) + throws Exception { + this(hiveConf, clusterType, false, null, null, + false, usePortsFromConf, "KERBEROS", false, true, + false, null, null); } public void start(Map confOverlay) throws Exception { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index ffce1d1..6705a98 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hive.common.log.ProgressMonitor; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.ql.MapRedStats; @@ -1589,10 +1590,11 @@ public void close() throws IOException { private void unCacheDataNucleusClassLoaders() { try { - Hive threadLocalHive = Hive.get(sessionConf); - if ((threadLocalHive != null) && (threadLocalHive.getMSC() != null) - && (threadLocalHive.getMSC().isLocalMetaStore())) { - if (sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL).equals(ObjectStore.class.getName())) { + boolean isLocalMetastore = + HiveConfUtil.isEmbeddedMetaStore(sessionConf.getVar(HiveConf.ConfVars.METASTOREURIS)); + if (isLocalMetastore) { + if (sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL) + .equals(ObjectStore.class.getName())) { ObjectStore.unCacheDataNucleusClassLoaders(); } } diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java index 00a7e74..444d741 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java @@ -125,6 +125,8 @@ private void cancelDelegationToken() throws HiveSQLException { if (hmsDelegationTokenStr != null) { try { Hive.get(getHiveConf()).cancelDelegationToken(hmsDelegationTokenStr); + hmsDelegationTokenStr = null; + getHiveConf().setVar(HiveConf.ConfVars.METASTORE_TOKEN_SIGNATURE, ""); } catch (HiveException e) { throw new HiveSQLException("Couldn't cancel delegation token", e); } diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java index d6dc079..326ef4e 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java @@ -152,8 +152,8 @@ private Object invokeOnTokenStore(String methName, Object[] params, Class ... break; case HIVESERVER2 : Object hiveObject = ((Class)handler) - .getMethod("get", org.apache.hadoop.conf.Configuration.class, java.lang.Class.class) - .invoke(handler, conf, DBTokenStore.class); + .getMethod("get") + .invoke(handler, null); tokenStore = ((Class)handler).getMethod("getMSC").invoke(hiveObject); break; default: diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java index 5299e18..f445b44 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java @@ -97,12 +97,12 @@ public synchronized long renewDelegationToken(String tokenStrForm) throws IOExce return renewToken(t, user); } - public synchronized String getDelegationToken(String renewer) throws IOException { - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - Text owner = new Text(ugi.getUserName()); + public synchronized String getDelegationToken(String ownerStr, String renewer) throws IOException { + UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser(); + Text owner = new Text(ownerStr); Text realUser = null; - if (ugi.getRealUser() != null) { - realUser = new Text(ugi.getRealUser().getUserName()); + if (currentUgi.getRealUser() != null) { + realUser = new Text(currentUgi.getRealUser().getUserName()); } DelegationTokenIdentifier ident = new DelegationTokenIdentifier(owner, new Text(renewer), realUser); diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java index b3e4a76..8d172fa 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java @@ -113,16 +113,15 @@ public String getDelegationToken(final String owner, final String renewer, Strin // in the case of proxy users, the getCurrentUser will return the // real user (for e.g. oozie) due to the doAs that happened just before the // server started executing the method getDelegationToken in the MetaStore - ownerUgi = UserGroupInformation.createProxyUser(owner, UserGroupInformation.getCurrentUser()); ProxyUsers.authorize(ownerUgi, remoteAddr, null); } - return ownerUgi.doAs(new PrivilegedExceptionAction() { - - @Override - public String run() throws IOException { - return secretManager.getDelegationToken(renewer); - } - }); + //if impersonation is turned on this called using the HiveSessionImplWithUGI + //using sessionProxy. so the currentUser will be the impersonated user here like oozie + //we cannot create a proxy user which represents Oozie's client user here since + //we cannot authenticate it using Kerberos/Digest. + //if impersonation is turned off, the current user is Hive which can open + //kerberos connections to HMS if required. + return secretManager.getDelegationToken(owner, renewer); } public String getDelegationTokenWithService(String owner, String renewer, String service, String remoteAddr)