diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index b7972838cfd..86d63709274 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ReservationId; @@ -381,12 +382,21 @@ private void loadRMDTSecretManagerState(RMState rmState) throws Exception { RMStateStoreUtils.readRMDelegationTokenIdentifierData(fsIn); RMDelegationTokenIdentifier identifier = identifierData.getTokenIdentifier(); - long renewDate = identifierData.getRenewDate(); - rmState.rmSecretManagerState.delegationTokenState.put(identifier, - renewDate); - if (LOG.isDebugEnabled()) { - LOG.debug("Loaded RMDelegationTokenIdentifier: " + identifier - + " renewDate=" + renewDate); + // do not restore already expired token, remove it from the store + if (identifier.getMaxDate() > Time.now()) { + long renewDate = identifierData.getRenewDate(); + rmState.rmSecretManagerState.delegationTokenState.put(identifier, + renewDate); + if (LOG.isDebugEnabled()) { + LOG.debug("Loaded RMDelegationTokenIdentifier: " + identifier + + " renewDate=" + renewDate); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Not recovering RMDelegationTokenIdentifier: " + + identifier + " already expired: " + identifier.getMaxDate()); + } + deleteFile(childNodePath); } } else { LOG.warn("Unknown file for recovering RMDelegationTokenSecretManager"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java index e7fb02fa7b5..54b591981aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java @@ -386,13 +386,22 @@ private int loadRMDTSecretManagerTokens(RMState state) throws IOException { RMDelegationTokenIdentifierData tokenData = loadDelegationToken( entry.getValue()); RMDelegationTokenIdentifier tokenId = tokenData.getTokenIdentifier(); - long renewDate = tokenData.getRenewDate(); - state.rmSecretManagerState.delegationTokenState.put(tokenId, - renewDate); - ++numTokens; - if (LOG.isDebugEnabled()) { - LOG.debug("Loaded RM delegation token from " + key - + ": tokenId=" + tokenId + ", renewDate=" + renewDate); + // do not restore already expired token, remove it from the store + if (tokenId.getMaxDate() > Time.now()) { + long renewDate = tokenData.getRenewDate(); + state.rmSecretManagerState.delegationTokenState.put(tokenId, + renewDate); + ++numTokens; + if (LOG.isDebugEnabled()) { + LOG.debug("Loaded RM delegation token from " + key + + ": tokenId=" + tokenId + ", renewDate=" + renewDate); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Not recovering RMDelegationTokenIdentifier: " + + tokenId + " already expired: " + tokenId.getMaxDate()); + } + removeRMDelegationTokenState(tokenId); } } } catch (DBException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index bd76a8c434b..9344b5bbc07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.util.Time; import org.apache.hadoop.util.ZKUtil; import org.apache.hadoop.util.curator.ZKCuratorManager; import org.apache.hadoop.util.curator.ZKCuratorManager.SafeTransaction; @@ -671,12 +672,21 @@ private void loadDelegationTokenFromNode(RMState rmState, String path) RMStateStoreUtils.readRMDelegationTokenIdentifierData(fsIn); RMDelegationTokenIdentifier identifier = identifierData.getTokenIdentifier(); - long renewDate = identifierData.getRenewDate(); - rmState.rmSecretManagerState.delegationTokenState.put(identifier, - renewDate); - if (LOG.isDebugEnabled()) { - LOG.debug("Loaded RMDelegationTokenIdentifier: " + identifier - + " renewDate=" + renewDate); + // do not restore already expired token, remove it from the store + if (identifier.getMaxDate() > Time.now()) { + long renewDate = identifierData.getRenewDate(); + rmState.rmSecretManagerState.delegationTokenState.put(identifier, + renewDate); + if (LOG.isDebugEnabled()) { + LOG.debug("Loaded RMDelegationTokenIdentifier: " + identifier + + " renewDate=" + renewDate); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Not recovering RMDelegationTokenIdentifier: " + + identifier + " already expired: " + identifier.getMaxDate()); + } + delete(path); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 3454d7265e8..d46f1641629 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -18,12 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -92,6 +87,7 @@ public static final Log LOG = LogFactory.getLog(RMStateStoreTestBase.class); + protected static final long DAY_MS = 24 * 60 * 60 * 1000; protected final long epoch = 10L; private final long epochRange = 10L; @@ -466,8 +462,9 @@ public void testRMDTSecretManagerStateStore( new Text("renewer1"), new Text("realuser1")); int sequenceNumber = 1111; dtId1.setSequenceNumber(sequenceNumber); - byte[] tokenBeforeStore = dtId1.getBytes(); Long renewDate1 = new Long(System.currentTimeMillis()); + dtId1.setMaxDate(renewDate1 + DAY_MS); + byte[] tokenBeforeStore = dtId1.getBytes(); store.storeRMDelegationToken(dtId1, renewDate1); modifyRMDelegationTokenState(); Map token1 = @@ -523,8 +520,18 @@ public void testRMDTSecretManagerStateStore( noKeyAndTokenSecretManagerState.getMasterKeyState()); Assert.assertEquals(sequenceNumber, noKeySecretManagerState.getDTSequenceNumber()); - store.close(); + RMDelegationTokenIdentifier tokenExp = new RMDelegationTokenIdentifier( + new Text("owner"), new Text("renewer"), new Text("realuser")); + Long renewDate = System.currentTimeMillis(); + tokenExp.setMaxDate(renewDate - DAY_MS); + tokenExp.setIssueDate(renewDate - DAY_MS); + tokenExp.setSequenceNumber(2222); + store.storeRMDelegationToken(tokenExp, renewDate); + RMDTSecretManagerState state = store.loadState().getRMDTSecretManagerState(); + assertFalse("Token should not exist but was found in Store", + state.getTokenState().containsKey(tokenExp.getSequenceNumber())); + store.close(); } protected Token generateAMRMToken( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 11be3b15ac4..4dd315f05fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -1245,6 +1245,7 @@ private RMDelegationTokenIdentifier storeUpdateAndVerifyDelegationToken( zkTester.delegationTokenExists(token, split)); token.setSequenceNumber(sequenceNumber); Long renewDate = System.currentTimeMillis(); + token.setMaxDate(renewDate + DAY_MS); zkTester.store.storeRMDelegationToken(token, renewDate); modifyRMDelegationTokenState(); tokensWithRenewal.put(token, renewDate); @@ -1366,9 +1367,10 @@ public void testDelegationTokenNodeWithSplitMultiple() throws Exception { new Text("renewer" + i), new Text("realuser" + i)); sequenceNumber = i; token.setSequenceNumber(sequenceNumber); + Long renewDate = System.currentTimeMillis(); + token.setMaxDate(renewDate + DAY_MS); assertFalse("Token should not exist but was found in ZooKeeper", zkTester.delegationTokenExists(token, 1)); - Long renewDate = System.currentTimeMillis(); store.storeRMDelegationToken(token, renewDate); modifyRMDelegationTokenState(); tokensWithRenewal.put(token, renewDate); @@ -1401,6 +1403,7 @@ public void testDelegationTokenNodeWithSplitMultiple() throws Exception { sequenceNumber = i; token.setSequenceNumber(sequenceNumber); Long renewDate = System.currentTimeMillis(); + token.setMaxDate(renewDate + DAY_MS); store.storeRMDelegationToken(token, renewDate); modifyRMDelegationTokenState(); tokensWithRenewal.put(token, renewDate); @@ -1562,4 +1565,35 @@ public void testAppSubmissionContextIsPrunedInFinalApplicationState() Collections.emptyMap(), ctx.getApplicationSchedulingPropertiesMap()); store.close(); } + + @Test + public void testRestoreExpiredToken() throws Exception { + int split = 0; + TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester(); + Configuration conf = createConfForDelegationTokenNodeSplit(split); + RMStateStore store = zkTester.getRMStateStore(conf); + RMDelegationTokenIdentifier tokenExp = new RMDelegationTokenIdentifier( + new Text("owner"), new Text("renewer"), new Text("realuser")); + Long renewDate = System.currentTimeMillis(); + tokenExp.setMaxDate(renewDate - DAY_MS); + tokenExp.setIssueDate(renewDate - DAY_MS); + tokenExp.setSequenceNumber(0); + store.storeRMDelegationToken(tokenExp, renewDate); + RMDelegationTokenIdentifier tokenValid = new RMDelegationTokenIdentifier( + new Text("owner"), new Text("renewer"), new Text("realuser")); + tokenValid.setMaxDate(renewDate + DAY_MS); + tokenValid.setIssueDate(renewDate); + tokenValid.setSequenceNumber(1); + store.storeRMDelegationToken(tokenValid, renewDate); + assertTrue("Token should exist but was not found in ZooKeeper", + zkTester.delegationTokenExists(tokenValid, split)); + assertTrue("Token should exist but was not found in ZooKeeper", + zkTester.delegationTokenExists(tokenExp, split)); + store.loadState(); + assertTrue("Token should exist but was not found in ZooKeeper", + zkTester.delegationTokenExists(tokenValid, split)); + assertFalse("Token should not exist but was found in ZooKeeper", + zkTester.delegationTokenExists(tokenExp, split)); + store.close(); + } }