diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index 7593b4e57dd..d9d0fa357e7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -300,7 +300,8 @@ protected void updateToken(TokenIdent ident, /** * This method is intended to be used for recovering persisted delegation - * tokens + * tokens. Tokens that have an unknown DelegationKey are + * marked as expired and automatically cleaned up. * This method must be called before this secret manager is activated (before * startThreads() is called) * @param identifier identifier read from persistent storage @@ -316,12 +317,16 @@ public synchronized void addPersistedDelegationToken( } int keyId = identifier.getMasterKeyId(); DelegationKey dKey = allKeys.get(keyId); + byte[] password = null; if (dKey == null) { - LOG.warn("No KEY found for persisted identifier " + LOG.warn("No KEY found for persisted identifier, expiring stored token " + formatTokenId(identifier)); - return; + // make sure the token is epxired + identifier.setMaxDate(0L); + renewDate = 0L; + } else { + password = createPassword(identifier.getBytes(), dKey.getKey()); } - byte[] password = createPassword(identifier.getBytes(), dKey.getKey()); if (identifier.getSequenceNumber() > getDelegationTokenSeqNum()) { setDelegationTokenSeqNum(identifier.getSequenceNumber()); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJHSDelegationTokenSecretManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJHSDelegationTokenSecretManager.java index f41bb3a98ff..f1521cc81f1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJHSDelegationTokenSecretManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJHSDelegationTokenSecretManager.java @@ -39,7 +39,7 @@ public class TestJHSDelegationTokenSecretManager { @Test - public void testRecovery() throws IOException { + public void testRecovery() throws Exception { Configuration conf = new Configuration(); HistoryServerStateStoreService store = new HistoryServerMemStateStoreService(); @@ -124,7 +124,16 @@ public void testRecovery() throws IOException { mgr.startThreads(); mgr.verifyToken(tokenId2, token2.getPassword()); mgr.verifyToken(tokenId3, token3.getPassword()); + // Set an unknown key ID: token should not be restored + tokenId3.setMasterKeyId(1000); + mgr.updateStoredToken(tokenId3, tokenRenewDate3); mgr.stopThreads(); + mgr = new JHSDelegationTokenSecretManagerForTest(store); + mgr.recover(store.loadState()); + mgr.startThreads(); + Thread.sleep(100); + assertFalse("token3 should be missing", + mgr.getAllTokens().containsKey(tokenId3)); } private static class JHSDelegationTokenSecretManagerForTest diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java index 2c52377995a..927b9661d28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java @@ -33,6 +33,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -164,6 +165,55 @@ public void testRemoveExpiredMasterKeyInRMStateStore() throws Exception { rm1.stop(); } + // Test removing token without key from state-store. + @Test(timeout = 15000) + public void testUnknownKeyTokensOnRecover() throws Exception { + final int masterID = 1234; + final int sequenceNumber = 1000; + + MemoryRMStateStore memStore = new MockMemoryRMStateStore(); + memStore.init(testConf); + // Need RM to get the secret manager and call recover + MockRM rm1 = new MyMockRM(testConf, memStore); + rm1.start(); + RMDelegationTokenSecretManager dtSecretManager = + rm1.getRMContext().getRMDelegationTokenSecretManager(); + // short cut to generate a basic token with unknown key + RMDelegationTokenIdentifier rmDT = new RMDelegationTokenIdentifier( + new Text("owner"), new Text("renewer"), new Text("realuser")); + // set a master key which is not used + rmDT.setMasterKeyId(masterID); + rmDT.setSequenceNumber(sequenceNumber); + final long tokenTime = Time.now() + 60000; + rmDT.setMaxDate(tokenTime); + dtSecretManager.storeNewToken(rmDT, tokenTime); + // give it time to process + Thread.sleep(100); + RMState rmState = memStore.getState(); + Assert.assertEquals("One token should have been stored", 1, + rmState.getRMDTSecretManagerState().getTokenState().size()); + Assert.assertTrue("Token should be in secret manager but is not", + rmState.getRMDTSecretManagerState().getTokenState().containsKey(rmDT)); + // Cannot recover while running: stop and clear + dtSecretManager.stopThreads(); + dtSecretManager.reset(); + Assert.assertFalse("Token should not be in secret manager but is found", + dtSecretManager.getAllTokens().containsKey(rmDT)); + dtSecretManager.recover(rmState); + Assert.assertTrue("Token should be in the state but is not found", + rmState.getRMDTSecretManagerState().getTokenState().containsKey(rmDT)); + dtSecretManager.getRenewDate(rmDT); + Assert.assertEquals("Token should have been expired but is not", 0L, + dtSecretManager.getRenewDate(rmDT)); + // The remover thread should immediately do its work, + // still give it some time to process + dtSecretManager.startThreads(); + Thread.sleep(100); + Assert.assertFalse("Token should not be in secret manager but it is", + rmState.getRMDTSecretManagerState().getTokenState().containsKey(rmDT)); + rm1.stop(); + } + class MyMockRM extends TestSecurityMockRM { public MyMockRM(Configuration conf, RMStateStore store) {