diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java index 2e5aa94..6d4fdfc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java @@ -115,7 +115,7 @@ public synchronized int getLocalAMRMTokenKeyId() { throw new YarnRuntimeException("Missing AMRM token for " + this.applicationAttemptId); } - keyId = this.amrmToken.decodeIdentifier().getKeyId(); + keyId = this.localToken.decodeIdentifier().getKeyId(); this.localTokenKeyId = keyId; } catch (IOException e) { throw new YarnRuntimeException("AMRM token decode error for " diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java index dc56090..5c2a9ae 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java @@ -342,9 +342,14 @@ private void updateAMRMTokens(AMRMTokenIdentifier amrmTokenIdentifier, // check to see if the RM has issued a new AMRMToken & accordingly update // the real ARMRMToken in the current context if (allocateResponse.getAMRMToken() != null) { + LOG.info("RM rolled master-key for amrm-tokens"); + org.apache.hadoop.yarn.api.records.Token token = allocateResponse.getAMRMToken(); + // Do not propagate this info back to AM + allocateResponse.setAMRMToken(null); + org.apache.hadoop.security.token.Token newTokenId = new org.apache.hadoop.security.token.Token( token.getIdentifier().array(), token.getPassword().array(), @@ -371,6 +376,9 @@ private void updateAMRMTokens(AMRMTokenIdentifier amrmTokenIdentifier, this.secretManager.createAndGetAMRMToken(pipeline .getApplicationAttemptId()); context.setLocalAMRMToken(localToken); + } else { + LOG.warn( + "AM did not use the newly issued local AMRMToken, resending it"); } allocateResponse diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java index c69313f..f584c94 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java @@ -108,6 +108,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.UpdatedContainer; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -297,10 +298,14 @@ public AllocateResponse allocate(AllocateRequest request) Log.getLog().info("Allocating containers: " + containerList.size() + " for application attempt: " + conf.get("AMRMTOKEN")); + + // Always issue a new AMRMToken as if RM rolled master key + Token newAMRMToken = Token.newInstance(new byte[0], "", new byte[0], ""); + return AllocateResponse.newInstance(0, new ArrayList(), containerList, new ArrayList(), null, AMCommand.AM_RESYNC, 1, null, - new ArrayList(), + new ArrayList(), newAMRMToken, new ArrayList()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java index 69b913a..7fffddf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java @@ -401,6 +401,9 @@ public Integer invoke(Integer testAppId) { AllocateResponse allocateResponse = allocate(appId, allocateRequest); Assert.assertNotNull("allocate() returned null response", allocateResponse); + Assert.assertNull( + "new AMRMToken from RM should have been nulled by AMRMProxyService", + allocateResponse.getAMRMToken()); containers.addAll(allocateResponse.getAllocatedContainers()); @@ -412,6 +415,9 @@ public Integer invoke(Integer testAppId) { allocate(appId, Records.newRecord(AllocateRequest.class)); Assert.assertNotNull("allocate() returned null response", allocateResponse); + Assert.assertNull( + "new AMRMToken from RM should have been nulled by AMRMProxyService", + allocateResponse.getAMRMToken()); containers.addAll(allocateResponse.getAllocatedContainers()); @@ -447,6 +453,9 @@ private void releaseContainersAndAssert(int appId, AllocateResponse allocateResponse = allocate(appId, allocateRequest); Assert.assertNotNull(allocateResponse); + Assert.assertNull( + "new AMRMToken from RM should have been nulled by AMRMProxyService", + allocateResponse.getAMRMToken()); // The way the mock resource manager is setup, it will return the containers // that were released in the response. This is done because the UAMs run @@ -467,6 +476,10 @@ private void releaseContainersAndAssert(int appId, allocateResponse = allocate(appId, Records.newRecord(AllocateRequest.class)); Assert.assertNotNull(allocateResponse); + Assert.assertNull( + "new AMRMToken from RM should have been nulled by AMRMProxyService", + allocateResponse.getAMRMToken()); + containersForReleasedContainerIds.addAll(allocateResponse .getAllocatedContainers());