commit c60104ca0b56409002da650b5166e2cd9897c96c Author: Vihang Karajgaonkar Date: Fri Sep 1 19:14:35 2017 -0700 HIVE-17368 : DBTokenStore fails to connect in Kerberos enabled remote HMS environment diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java index bbec37eea76517e9d42e60b26d85cd0b22965cc9..c02879d9bfd023a24d2fefcded04cbf418fd72a6 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java @@ -51,6 +51,7 @@ public static String HIVE_TEST_USER_2 = "user2"; public static String HIVE_TEST_SUPER_USER = "superuser"; public static String AUTHENTICATION_TYPE = "KERBEROS"; + private static final String HIVE_METASTORE_SERVICE_PRINCIPAL = "hive"; private final MiniKdc miniKdc; private final File workDir; @@ -204,6 +205,39 @@ public static MiniHS2 getMiniHS2WithKerbWithRemoteHMS(MiniHiveKdc miniHiveKdc, H return getMiniHS2WithKerbWithRemoteHMS(miniHiveKdc, hiveConf, AUTHENTICATION_TYPE); } + public static MiniHS2 getMiniHS2WithKerbWithRemoteHMSWithKerb(MiniHiveKdc miniHiveKdc, + HiveConf hiveConf) throws Exception { + return getMiniHS2WithKerbWithRemoteHMSWithKerb(miniHiveKdc, hiveConf, AUTHENTICATION_TYPE); + } + + /** + * Create a MiniHS2 with the hive service principal and keytab in MiniHiveKdc. It uses remote HMS + * and can support a different Sasl authType. It creates a metastore service principal and keytab + * which can be used for secure HMS + * @param miniHiveKdc + * @param hiveConf + * @param authenticationType + * @return new MiniHS2 instance + * @throws Exception + */ + private static MiniHS2 getMiniHS2WithKerbWithRemoteHMSWithKerb(MiniHiveKdc miniHiveKdc, + HiveConf hiveConf, String authenticationType) throws Exception { + String hivePrincipal = + miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL); + String hiveKeytab = miniHiveKdc.getKeyTabFile( + miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL)); + + String hiveMetastorePrincipal = + miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_METASTORE_SERVICE_PRINCIPAL); + String hiveMetastoreKeytab = miniHiveKdc.getKeyTabFile( + miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_METASTORE_SERVICE_PRINCIPAL)); + + return new MiniHS2.Builder().withConf(hiveConf) + .withSecureRemoteMetastore(hiveMetastorePrincipal, hiveMetastoreKeytab). + withMiniKdc(hivePrincipal, hiveKeytab).withAuthenticationType(authenticationType) + .build(); + } + /** * Create a MiniHS2 with the hive service principal and keytab in MiniHiveKdc. It uses remote HMS * and can support a different Sasl authType diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java index d690aaa673a50785561750f4f461ec867b6f0abc..fc1dc493d7fe4512a6fdf9d4ddb41512697e63f9 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java @@ -34,7 +34,11 @@ public static void beforeTest() throws Exception { HiveConf hiveConf = new HiveConf(); hiveConf.setVar(ConfVars.METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS, "org.apache.hadoop.hive.thrift.DBTokenStore"); miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf); - miniHS2 = MiniHiveKdc.getMiniHS2WithKerbWithRemoteHMS(miniHiveKdc, hiveConf); + miniHS2 = MiniHiveKdc.getMiniHS2WithKerbWithRemoteHMSWithKerb(miniHiveKdc, hiveConf); miniHS2.start(confOverlay); + String metastorePrincipal = miniHS2.getConfProperty(ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname); + String hs2Principal = miniHS2.getConfProperty(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL.varname); + String hs2KeyTab = miniHS2.getConfProperty(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB.varname); + System.out.println("HS2 principal : " + hs2Principal + " HS2 keytab : " + hs2KeyTab + " Metastore principal : " + metastorePrincipal); } } \ No newline at end of file diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java index 36a9ea830a62496351103bde143bc9dd22c9ba23..7e5005cdcfd0b63693bf19edd77bd803658a7376 100644 --- a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java +++ b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/thrift/TestHadoopAuthBridge23.java @@ -169,7 +169,7 @@ public void testDelegationTokenSharedStore() throws Exception { tokenManager.startThreads(); tokenManager.stopThreads(); - String tokenStrForm = tokenManager.getDelegationToken(clientUgi.getShortUserName()); + String tokenStrForm = tokenManager.getDelegationToken(clientUgi.getShortUserName(), clientUgi.getShortUserName()); Token t= new Token(); t.decodeFromUrlString(tokenStrForm); diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 71f9640ad217ad60377720489e1ccc71506e51d7..ebc4c10b6408bc8693090a7fd76f6f08d7803918 100644 --- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -72,6 +72,7 @@ private final String serverPrincipal; private final boolean isMetastoreRemote; private final boolean cleanupLocalDirOnStartup; + private final boolean isMetastoreSecure; private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; public enum MiniClusterType { @@ -93,6 +94,9 @@ private String authType = "KERBEROS"; private boolean isHA = false; private boolean cleanupLocalDirOnStartup = true; + private boolean isMetastoreSecure; + private String metastoreServerPrincipal; + private String metastoreServerKeyTab; public Builder() { } @@ -119,6 +123,14 @@ public Builder withRemoteMetastore() { return this; } + public Builder withSecureRemoteMetastore(String metastoreServerPrincipal, String metastoreServerKeyTab) { + this.isMetastoreRemote = true; + this.isMetastoreSecure = true; + this.metastoreServerPrincipal = metastoreServerPrincipal; + this.metastoreServerKeyTab = metastoreServerKeyTab; + return this; + } + public Builder withConf(HiveConf hiveConf) { this.hiveConf = hiveConf; return this; @@ -153,7 +165,8 @@ public MiniHS2 build() throws Exception { hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_BINARY_MODE); } return new MiniHS2(hiveConf, miniClusterType, useMiniKdc, serverPrincipal, serverKeytab, - isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup); + isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup, + isMetastoreSecure, metastoreServerPrincipal, metastoreServerKeyTab); } } @@ -191,7 +204,10 @@ public boolean isUseMiniKdc() { private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useMiniKdc, String serverPrincipal, String serverKeytab, boolean isMetastoreRemote, - boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup) throws Exception { + boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup, + boolean isMetastoreSecure, + String metastoreServerPrincipal, + String metastoreKeyTab) throws Exception { // Always use localhost for hostname as some tests like SSL CN validation ones // are tied to localhost being present in the certificate name super( @@ -208,6 +224,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM this.useMiniKdc = useMiniKdc; this.serverPrincipal = serverPrincipal; this.isMetastoreRemote = isMetastoreRemote; + this.isMetastoreSecure = isMetastoreSecure; this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup; baseDir = getBaseDir(); localFS = FileSystem.getLocal(hiveConf); @@ -261,9 +278,15 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB, serverKeytab); hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, authType); } - String metaStoreURL = - "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator - + "test_metastore;create=true"; + + String metaStoreURL = "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator + + "test_metastore;create=true"; + + if (isMetastoreSecure) { + hiveConf.setVar(ConfVars.METASTORE_KERBEROS_PRINCIPAL, metastoreServerPrincipal); + hiveConf.setVar(ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, metastoreKeyTab); + hiveConf.setBoolVar(ConfVars.METASTORE_USE_THRIFT_SASL, true); + } fs.mkdirs(baseFsDir); Path wareHouseDir = new Path(baseFsDir, "warehouse"); @@ -301,10 +324,11 @@ public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType) throws Exception this(hiveConf, clusterType, false); } - public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, - boolean usePortsFromConf) throws Exception { - this(hiveConf, clusterType, false, null, null, false, usePortsFromConf, - "KERBEROS", false, true); + public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsFromConf) + throws Exception { + this(hiveConf, clusterType, false, null, null, + false, usePortsFromConf, "KERBEROS", false, true, + false, null, null); } public void start(Map confOverlay) throws Exception { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index ffce1d1aec8728840bb8ef726db1b600a9aeef38..6705a98bb2aaa309b51db3b6dee7ff9dc6009e6e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hive.common.log.ProgressMonitor; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.ql.MapRedStats; @@ -1589,10 +1590,11 @@ public void close() throws IOException { private void unCacheDataNucleusClassLoaders() { try { - Hive threadLocalHive = Hive.get(sessionConf); - if ((threadLocalHive != null) && (threadLocalHive.getMSC() != null) - && (threadLocalHive.getMSC().isLocalMetaStore())) { - if (sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL).equals(ObjectStore.class.getName())) { + boolean isLocalMetastore = + HiveConfUtil.isEmbeddedMetaStore(sessionConf.getVar(HiveConf.ConfVars.METASTOREURIS)); + if (isLocalMetastore) { + if (sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL) + .equals(ObjectStore.class.getName())) { ObjectStore.unCacheDataNucleusClassLoaders(); } } diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java index 00a7e742cabd2fc36faa464b29250b5a6a9b1159..444d741cd86c4e3f196fe6b496bb3416cf75ddc7 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java @@ -125,6 +125,8 @@ private void cancelDelegationToken() throws HiveSQLException { if (hmsDelegationTokenStr != null) { try { Hive.get(getHiveConf()).cancelDelegationToken(hmsDelegationTokenStr); + hmsDelegationTokenStr = null; + getHiveConf().setVar(HiveConf.ConfVars.METASTORE_TOKEN_SIGNATURE, ""); } catch (HiveException e) { throw new HiveSQLException("Couldn't cancel delegation token", e); } diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java index d6dc0796e77591d3afca8dbd29c3aa0eff255dd0..326ef4e99ca6ab7540508b258cc306d00e26758c 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java @@ -152,8 +152,8 @@ private Object invokeOnTokenStore(String methName, Object[] params, Class ... break; case HIVESERVER2 : Object hiveObject = ((Class)handler) - .getMethod("get", org.apache.hadoop.conf.Configuration.class, java.lang.Class.class) - .invoke(handler, conf, DBTokenStore.class); + .getMethod("get") + .invoke(handler, null); tokenStore = ((Class)handler).getMethod("getMSC").invoke(hiveObject); break; default: diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java index 5299e18743aa45c539287b335f95e8ce8df0fc35..f445b44ba60b74e63cc223d679facdf688caf98b 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java @@ -97,12 +97,12 @@ public synchronized long renewDelegationToken(String tokenStrForm) throws IOExce return renewToken(t, user); } - public synchronized String getDelegationToken(String renewer) throws IOException { - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - Text owner = new Text(ugi.getUserName()); + public synchronized String getDelegationToken(String ownerStr, String renewer) throws IOException { + UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser(); + Text owner = new Text(ownerStr); Text realUser = null; - if (ugi.getRealUser() != null) { - realUser = new Text(ugi.getRealUser().getUserName()); + if (currentUgi.getRealUser() != null) { + realUser = new Text(currentUgi.getRealUser().getUserName()); } DelegationTokenIdentifier ident = new DelegationTokenIdentifier(owner, new Text(renewer), realUser); diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java index b3e4a7608282be603e79d1d101679e239a5219b0..d33968fccf3008a2cefe6eade18f4b52c697e58b 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/HiveDelegationTokenManager.java @@ -116,13 +116,13 @@ public String getDelegationToken(final String owner, final String renewer, Strin ownerUgi = UserGroupInformation.createProxyUser(owner, UserGroupInformation.getCurrentUser()); ProxyUsers.authorize(ownerUgi, remoteAddr, null); } - return ownerUgi.doAs(new PrivilegedExceptionAction() { - - @Override - public String run() throws IOException { - return secretManager.getDelegationToken(renewer); - } - }); + //if impersonation is turned on this called using the HiveSessionImplWithUGI + //using sessionProxy. so the currentUser will be the impersonated user here like oozie + //we cannot create a proxy user which represents Oozie's client user here since + //we cannot authenticate it using Kerberos/Digest. + //if impersonation is turned off, the current user is Hive which can open + //kerberos connections to HMS if required. + return secretManager.getDelegationToken(owner, renewer); } public String getDelegationTokenWithService(String owner, String renewer, String service, String remoteAddr)