diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index fd12f11..96a4e0d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -43,6 +43,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import com.google.common.base.Throwables; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -53,6 +54,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.service.AbstractService; @@ -459,6 +461,17 @@ private void handleAppSubmitEvent(AbstractDelegationTokenRenewerAppEvent evt) try { renewToken(dttr); } catch (IOException ioe) { + if (ioe instanceof SecretManager.InvalidToken + && dttr.maxDate < Time.now() + && evt instanceof DelegationTokenRenewerAppRecoverEvent + && token.getKind().equals(HDFS_DELEGATION_KIND)) { + LOG.info("Failed to renew hdfs token " + dttr + + " on recovery as it expired, requesting new hdfs token for " + + applicationId + ", user=" + evt.getUser(), ioe); + requestNewHdfsDelegationToken(Arrays.asList(applicationId), + evt.getUser(), evt.shouldCancelAtEnd()); + continue; + } throw new IOException("Failed to renew token: " + dttr.token, ioe); } } @@ -586,8 +599,7 @@ public Long run() throws Exception { } catch (InterruptedException e) { throw new IOException(e); } - LOG.info("Renewed delegation-token= [" + dttr + "], for " - + dttr.referringAppIds); + LOG.info("Renewed delegation-token= [" + dttr + "]"); } // Request new hdfs token if the token is about to expire, and remove the old @@ -912,8 +924,8 @@ private void handleDTRenewerAppRecoverEvent( // Setup tokens for renewal during recovery DelegationTokenRenewer.this.handleAppSubmitEvent(event); } catch (Throwable t) { - LOG.warn( - "Unable to add the application to the delegation token renewer.", t); + LOG.warn("Unable to add the application to the delegation token" + + " renewer on recovery.", t); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 1bfac8d..fd61cdf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -84,6 +84,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.TestSecurityMockRM; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; @@ -968,6 +969,87 @@ public Boolean get() { Assert.assertTrue(appCredentials.getAllTokens().contains(expectedToken)); } + + // 1. token is expired before app completes. + // 2. RM shutdown. + // 3. When RM recovers the app, token renewal will fail as token expired. + // RM should request a new token and sent it to NM for log-aggregation. + @Test + public void testRMRestartWithExpiredToken() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); + conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); + UserGroupInformation.setConfiguration(conf); + + // create Token1: + Text userText1 = new Text("user1"); + DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(userText1, + new Text("renewer1"), userText1); + final Token token1 = new Token<>(dtId1.getBytes(), + "password1".getBytes(), dtId1.getKind(), new Text("service1")); + Credentials credentials = new Credentials(); + credentials.addToken(userText1, token1); + + MemoryRMStateStore memStore = new MemoryRMStateStore(); + memStore.init(conf); + MockRM rm1 = new TestSecurityMockRM(conf, memStore); + rm1.start(); + RMApp app = rm1.submitApp(200, "name", "user", + new HashMap(), false, "default", 1, + credentials); + + // create token2 + Text userText2 = new Text("user2"); + DelegationTokenIdentifier dtId2 = + new DelegationTokenIdentifier(userText1, new Text("renewer2"), + userText2); + final Token expectedToken = + new Token(dtId2.getBytes(), + "password2".getBytes(), dtId2.getKind(), new Text("service2")); + MockRM rm2 = new TestSecurityMockRM(conf, memStore) { + @Override + protected DelegationTokenRenewer createDelegationTokenRenewer() { + return new DelegationTokenRenewer() { + + @Override + protected void renewToken(final DelegationTokenToRenew dttr) + throws IOException { + if (dttr.token.equals(expectedToken)) { + super.renewToken(dttr); + } else { + throw new InvalidToken("Failed to renew"); + } + } + + @Override + protected Token[] obtainSystemTokensForUser(String user, + final Credentials credentials) throws IOException { + credentials.addToken(expectedToken.getService(), expectedToken); + return new Token[] { expectedToken }; + } + }; + } + }; + rm2.start(); + + // check nm can retrieve the token + final MockNM nm1 = + new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService()); + nm1.registerNode(); + NodeHeartbeatResponse response = nm1.nodeHeartbeat(true); + ByteBuffer tokenBuffer = + response.getSystemCredentialsForApps().get(app.getApplicationId()); + Assert.assertNotNull(tokenBuffer); + Credentials appCredentials = new Credentials(); + DataInputByteBuffer buf = new DataInputByteBuffer(); + tokenBuffer.rewind(); + buf.reset(tokenBuffer); + appCredentials.readTokenStorageStream(buf); + Assert.assertTrue(appCredentials.getAllTokens().contains(expectedToken)); + } + // YARN will get the token for the app submitted without the delegation token. @Test public void testAppSubmissionWithoutDelegationToken() throws Exception { @@ -1158,4 +1240,5 @@ public void testCancelWithMultipleAppSubmissions() throws Exception{ Assert.assertTrue(dttr.isTimerCancelled()); Assert.assertTrue(Renewer.cancelled); } + }