commit b01a6dc8f3f399e01d8a689e55da173724f7f43f Author: Vihang Karajgaonkar Date: Fri Sep 1 19:14:35 2017 -0700 HIVE-17368 : DBTokenStore fails to connect in Kerberos enabled remote HMS environment (Vihang Karajgaonakar, reviewed by Aihua Xu and Janaki Lahorani) diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java index bbec37eea76517e9d42e60b26d85cd0b22965cc9..c02879d9bfd023a24d2fefcded04cbf418fd72a6 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java @@ -51,6 +51,7 @@ public static String HIVE_TEST_USER_2 = "user2"; public static String HIVE_TEST_SUPER_USER = "superuser"; public static String AUTHENTICATION_TYPE = "KERBEROS"; + private static final String HIVE_METASTORE_SERVICE_PRINCIPAL = "hive"; private final MiniKdc miniKdc; private final File workDir; @@ -204,6 +205,39 @@ public static MiniHS2 getMiniHS2WithKerbWithRemoteHMS(MiniHiveKdc miniHiveKdc, H return getMiniHS2WithKerbWithRemoteHMS(miniHiveKdc, hiveConf, AUTHENTICATION_TYPE); } + public static MiniHS2 getMiniHS2WithKerbWithRemoteHMSWithKerb(MiniHiveKdc miniHiveKdc, + HiveConf hiveConf) throws Exception { + return getMiniHS2WithKerbWithRemoteHMSWithKerb(miniHiveKdc, hiveConf, AUTHENTICATION_TYPE); + } + + /** + * Create a MiniHS2 with the hive service principal and keytab in MiniHiveKdc. It uses remote HMS + * and can support a different Sasl authType. It creates a metastore service principal and keytab + * which can be used for secure HMS + * @param miniHiveKdc + * @param hiveConf + * @param authenticationType + * @return new MiniHS2 instance + * @throws Exception + */ + private static MiniHS2 getMiniHS2WithKerbWithRemoteHMSWithKerb(MiniHiveKdc miniHiveKdc, + HiveConf hiveConf, String authenticationType) throws Exception { + String hivePrincipal = + miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL); + String hiveKeytab = miniHiveKdc.getKeyTabFile( + miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL)); + + String hiveMetastorePrincipal = + miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_METASTORE_SERVICE_PRINCIPAL); + String hiveMetastoreKeytab = miniHiveKdc.getKeyTabFile( + miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_METASTORE_SERVICE_PRINCIPAL)); + + return new MiniHS2.Builder().withConf(hiveConf) + .withSecureRemoteMetastore(hiveMetastorePrincipal, hiveMetastoreKeytab). + withMiniKdc(hivePrincipal, hiveKeytab).withAuthenticationType(authenticationType) + .build(); + } + /** * Create a MiniHS2 with the hive service principal and keytab in MiniHiveKdc. It uses remote HMS * and can support a different Sasl authType diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java index 869523200af53535c33f01e1251a84c83a4986d3..ab25c3b6fba50f4324fe51c95e8ef5c160802482 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java @@ -40,7 +40,11 @@ public static void beforeTest() throws Exception { //using old config value tests backwards compatibility hiveConf.setVar(ConfVars.METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS, "org.apache.hadoop.hive.thrift.DBTokenStore"); miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf); - miniHS2 = MiniHiveKdc.getMiniHS2WithKerbWithRemoteHMS(miniHiveKdc, hiveConf); + miniHS2 = MiniHiveKdc.getMiniHS2WithKerbWithRemoteHMSWithKerb(miniHiveKdc, hiveConf); miniHS2.start(confOverlay); + String metastorePrincipal = miniHS2.getConfProperty(ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname); + String hs2Principal = miniHS2.getConfProperty(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL.varname); + String hs2KeyTab = miniHS2.getConfProperty(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB.varname); + System.out.println("HS2 principal : " + hs2Principal + " HS2 keytab : " + hs2KeyTab + " Metastore principal : " + metastorePrincipal); } } \ No newline at end of file diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java index cedc93310655e78a79267491961d38388b757550..8094f924bfca5ea10c45793febe407651fb17e6e 100644 --- a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java +++ b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java @@ -168,7 +168,8 @@ public void testDelegationTokenSharedStore() throws Exception { tokenManager.startThreads(); tokenManager.stopThreads(); - String tokenStrForm = tokenManager.getDelegationToken(clientUgi.getShortUserName()); + String tokenStrForm = + tokenManager.getDelegationToken(clientUgi.getShortUserName(), clientUgi.getShortUserName()); Token t= new Token(); t.decodeFromUrlString(tokenStrForm); diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 5702de289082c922c7405c1e07a1f32e1ce11382..e554cc83e513b9a40183510fbc1338c52c58c1f0 100644 --- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -73,6 +73,7 @@ private final String serverPrincipal; private final boolean isMetastoreRemote; private final boolean cleanupLocalDirOnStartup; + private final boolean isMetastoreSecure; private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; public enum MiniClusterType { @@ -94,6 +95,9 @@ private String authType = "KERBEROS"; private boolean isHA = false; private boolean cleanupLocalDirOnStartup = true; + private boolean isMetastoreSecure; + private String metastoreServerPrincipal; + private String metastoreServerKeyTab; public Builder() { } @@ -120,6 +124,14 @@ public Builder withRemoteMetastore() { return this; } + public Builder withSecureRemoteMetastore(String metastoreServerPrincipal, String metastoreServerKeyTab) { + this.isMetastoreRemote = true; + this.isMetastoreSecure = true; + this.metastoreServerPrincipal = metastoreServerPrincipal; + this.metastoreServerKeyTab = metastoreServerKeyTab; + return this; + } + public Builder withConf(HiveConf hiveConf) { this.hiveConf = hiveConf; return this; @@ -154,7 +166,8 @@ public MiniHS2 build() throws Exception { hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_BINARY_MODE); } return new MiniHS2(hiveConf, miniClusterType, useMiniKdc, serverPrincipal, serverKeytab, - isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup); + isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup, + isMetastoreSecure, metastoreServerPrincipal, metastoreServerKeyTab); } } @@ -192,7 +205,10 @@ public boolean isUseMiniKdc() { private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useMiniKdc, String serverPrincipal, String serverKeytab, boolean isMetastoreRemote, - boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup) throws Exception { + boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup, + boolean isMetastoreSecure, + String metastoreServerPrincipal, + String metastoreKeyTab) throws Exception { // Always use localhost for hostname as some tests like SSL CN validation ones // are tied to localhost being present in the certificate name super( @@ -209,6 +225,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM this.useMiniKdc = useMiniKdc; this.serverPrincipal = serverPrincipal; this.isMetastoreRemote = isMetastoreRemote; + this.isMetastoreSecure = isMetastoreSecure; this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup; baseDir = getBaseDir(); localFS = FileSystem.getLocal(hiveConf); @@ -262,9 +279,15 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB, serverKeytab); hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, authType); } - String metaStoreURL = - "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator - + "test_metastore;create=true"; + + String metaStoreURL = "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator + + "test_metastore;create=true"; + + if (isMetastoreSecure) { + hiveConf.setVar(ConfVars.METASTORE_KERBEROS_PRINCIPAL, metastoreServerPrincipal); + hiveConf.setVar(ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, metastoreKeyTab); + hiveConf.setBoolVar(ConfVars.METASTORE_USE_THRIFT_SASL, true); + } fs.mkdirs(baseFsDir); Path wareHouseDir = new Path(baseFsDir, "warehouse"); @@ -302,10 +325,11 @@ public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType) throws Exception this(hiveConf, clusterType, false); } - public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, - boolean usePortsFromConf) throws Exception { - this(hiveConf, clusterType, false, null, null, false, usePortsFromConf, - "KERBEROS", false, true); + public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsFromConf) + throws Exception { + this(hiveConf, clusterType, false, null, null, + false, usePortsFromConf, "KERBEROS", false, true, + false, null, null); } public void start(Map confOverlay) throws Exception { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 2bdb7192cdb738273b0535c3ed7f5eb18f306a7c..bb6ddc6fa4667ac0e30994d0f9ee8b969542383c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hive.common.log.ProgressMonitor; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.cache.CachedStore; @@ -1754,12 +1755,15 @@ public void close() throws IOException { private void unCacheDataNucleusClassLoaders() { try { - Hive threadLocalHive = Hive.get(sessionConf); - if ((threadLocalHive != null) && (threadLocalHive.getMSC() != null) - && (threadLocalHive.getMSC().isLocalMetaStore())) { - if (sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL).equals(ObjectStore.class.getName()) - || sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL).equals(CachedStore.class.getName()) && - sessionConf.getVar(ConfVars.METASTORE_CACHED_RAW_STORE_IMPL).equals(ObjectStore.class.getName())) { + boolean isLocalMetastore = + HiveConfUtil.isEmbeddedMetaStore(sessionConf.getVar(HiveConf.ConfVars.METASTOREURIS)); + if (isLocalMetastore) { + if (sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL) + .equals(ObjectStore.class.getName()) || + sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL) + .equals(CachedStore.class.getName()) && sessionConf + .getVar(ConfVars.METASTORE_CACHED_RAW_STORE_IMPL) + .equals(ObjectStore.class.getName())) { ObjectStore.unCacheDataNucleusClassLoaders(); } } diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java index 8000a5bb46af714b4b4c5c933484e35a4cd054f9..8975aeea7026ae8e7c96361421d0c9a0063a2277 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java @@ -124,6 +124,8 @@ private void cancelDelegationToken() throws HiveSQLException { if (hmsDelegationTokenStr != null) { try { Hive.get(getHiveConf()).cancelDelegationToken(hmsDelegationTokenStr); + hmsDelegationTokenStr = null; + getHiveConf().setVar(HiveConf.ConfVars.METASTORE_TOKEN_SIGNATURE, ""); } catch (HiveException e) { throw new HiveSQLException("Couldn't cancel delegation token", e); } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DBTokenStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DBTokenStore.java index 2f260100f65b836798fdaf6d55f9a241fc9e4b3a..ad6d751dc6ddf3b38fb6b097d815f8d429d13097 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DBTokenStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DBTokenStore.java @@ -152,9 +152,7 @@ private Object invokeOnTokenStore(String methName, Object[] params, Class ... tokenStore = handler.getClass().getMethod("getMS").invoke(handler); break; case HIVESERVER2: - Object hiveObject = ((Class) handler) - .getMethod("get", org.apache.hadoop.conf.Configuration.class, java.lang.Class.class) - .invoke(handler, conf, DBTokenStore.class); + Object hiveObject = ((Class) handler).getMethod("get").invoke(handler, null); tokenStore = ((Class) handler).getMethod("getMSC").invoke(hiveObject); break; default: diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java index aae96a5dfedfd99b7513f7f8582a1782c998504e..a719f06ec2ee5b9bd98145dc32f9f4e0c43f83a1 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java @@ -97,12 +97,15 @@ public synchronized long renewDelegationToken(String tokenStrForm) throws IOExce return renewToken(t, user); } - public synchronized String getDelegationToken(String renewer) throws IOException { - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - Text owner = new Text(ugi.getUserName()); + public synchronized String getDelegationToken(final String ownerStr, final String renewer) throws IOException { + if (ownerStr == null) { + throw new RuntimeException("Delegation token owner is null"); + } + Text owner = new Text(ownerStr); Text realUser = null; - if (ugi.getRealUser() != null) { - realUser = new Text(ugi.getRealUser().getUserName()); + UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser(); + if (currentUgi.getUserName() != null) { + realUser = new Text(currentUgi.getUserName()); } DelegationTokenIdentifier ident = new DelegationTokenIdentifier(owner, new Text(renewer), realUser); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java index 3cfdd8a15591ce18bb3bd5a07bd8e17d875cab89..8d84fd7f323af1de4ac994863d81e5b06be5bcce 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java @@ -99,13 +99,14 @@ public String getDelegationToken(final String owner, final String renewer, Strin ownerUgi = UserGroupInformation.createProxyUser(owner, UserGroupInformation.getCurrentUser()); ProxyUsers.authorize(ownerUgi, remoteAddr, null); } - return ownerUgi.doAs(new PrivilegedExceptionAction() { - - @Override - public String run() throws IOException { - return secretManager.getDelegationToken(renewer); - } - }); + //if impersonation is turned on this called using the HiveSessionImplWithUGI + //using sessionProxy. so the currentUser will be the impersonated user here eg. oozie + //we cannot create a proxy user which represents Oozie's client user here since + //we cannot authenticate it using Kerberos/Digest. We trust the user which opened + //session using Kerberos in this case. + //if impersonation is turned off, the current user is Hive which can open + //kerberos connections to HMS if required. + return secretManager.getDelegationToken(owner, renewer); } public String getDelegationTokenWithService(String owner, String renewer, String service, String remoteAddr)