diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index 0eb5a3d..c9f3541 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -190,6 +190,9 @@ public synchronized void storeRMDelegationTokenAndSequenceNumberState( } rmDTState.put(rmDTIdentifier, renewDate); state.rmSecretManagerState.dtSequenceNumber = latestSequenceNumber; + LOG.info("Store RMDT with sequence number " + + rmDTIdentifier.getSequenceNumber() + + ". And the latest sequence number is " + latestSequenceNumber); } @Override @@ -198,6 +201,8 @@ public synchronized void removeRMDelegationTokenState( Map rmDTState = state.rmSecretManagerState.getTokenState(); rmDTState.remove(rmDTIdentifier); + LOG.info("Remove RMDT with sequence number " + + rmDTIdentifier.getSequenceNumber()); } @Override @@ -207,6 +212,8 @@ protected void updateRMDelegationTokenAndSequenceNumberInternal( removeRMDelegationTokenState(rmDTIdentifier); storeRMDelegationTokenAndSequenceNumberState( rmDTIdentifier, renewDate, latestSequenceNumber); + LOG.info("Update RMDT with sequence number " + + rmDTIdentifier.getSequenceNumber()); } @Override @@ -223,12 +230,14 @@ public synchronized void storeRMDTMasterKeyState(DelegationKey delegationKey) throw e; } state.getRMDTSecretManagerState().getMasterKeyState().add(delegationKey); - LOG.info("rmDTMasterKeyState SIZE: " + rmDTMasterKeyState.size()); + LOG.info("Store RMDT master key with key id: " + delegationKey.getKeyId() + + ". Currently rmDTMasterKeyState size: " + rmDTMasterKeyState.size()); } @Override public synchronized void removeRMDTMasterKeyState(DelegationKey delegationKey) throws Exception { + LOG.info("Remove RMDT master key with key id: " + delegationKey.getKeyId()); Set rmDTMasterKeyState = state.rmSecretManagerState.getMasterKeyState(); rmDTMasterKeyState.remove(delegationKey); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java index cf11e48..5b5b3be 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java @@ -67,6 +67,7 @@ public void setup() { conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); } + // Test the DT mast key in the state-store when the mast key is being rolled. @Test(timeout = 15000) public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception { MemoryRMStateStore memStore = new MemoryRMStateStore(); @@ -92,9 +93,6 @@ public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception { Set expiringKeys = new HashSet(); expiringKeys.addAll(dtSecretManager.getAllMasterKeys()); - // record the current key - DelegationKey oldCurrentKey = - ((TestRMDelegationTokenSecretManager) dtSecretManager).getCurrentKey(); // request to generate a RMDelegationToken GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class); @@ -107,29 +105,25 @@ public void testRMDTMasterKeyStateOnRollingMasterKey() throws Exception { ConverterUtils.convertFromYarn(delegationToken, (Text) null); RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier(); - // wait for the first rollMasterKey + // For all keys that still remain in memory, we should have them stored + // in state-store also. while (((TestRMDelegationTokenSecretManager) dtSecretManager).numUpdatedKeys - .get() < 1){ - Thread.sleep(200); + .get() < 3) { + ((TestRMDelegationTokenSecretManager) dtSecretManager) + .checkCurrentKeyInStateStore(rmDTMasterKeyState); + Thread.sleep(100); } - // assert old-current-key and new-current-key exist - Assert.assertTrue(rmDTMasterKeyState.contains(oldCurrentKey)); - DelegationKey newCurrentKey = - ((TestRMDelegationTokenSecretManager) dtSecretManager).getCurrentKey(); - Assert.assertTrue(rmDTMasterKeyState.contains(newCurrentKey)); - - // wait for token to expire + // wait for token to expire and remove from state-store // rollMasterKey is called every 1 second. - while (((TestRMDelegationTokenSecretManager) dtSecretManager).numUpdatedKeys - .get() < 6) { - Thread.sleep(200); + int count = 0; + while (rmDTState.containsKey(dtId1) && count < 100) { + Thread.sleep(100); } - - Assert.assertFalse(rmDTState.containsKey(dtId1)); rm1.stop(); } + // Test all expired keys are removed from state-store. @Test(timeout = 15000) public void testRemoveExpiredMasterKeyInRMStateStore() throws Exception { MemoryRMStateStore memStore = new MemoryRMStateStore(); @@ -205,10 +199,13 @@ protected void storeNewMasterKey(DelegationKey newKey) { numUpdatedKeys.incrementAndGet(); } - public DelegationKey getCurrentKey() { + public synchronized DelegationKey checkCurrentKeyInStateStore( + Set rmDTMasterKeyState) { for (int keyId : allKeys.keySet()) { if (keyId == currentId) { - return allKeys.get(keyId); + DelegationKey currentKey = allKeys.get(keyId); + Assert.assertTrue(rmDTMasterKeyState.contains(currentKey)); + return currentKey; } } return null;