diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java index 99495d7..bc2d7c5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java @@ -44,6 +44,7 @@ public static final Text KIND_NAME = new Text("YARN_AM_RM_TOKEN"); private ApplicationAttemptId applicationAttemptId; + private int keyId = Integer.MIN_VALUE; public AMRMTokenIdentifier() { } @@ -53,6 +54,13 @@ public AMRMTokenIdentifier(ApplicationAttemptId appAttemptId) { this.applicationAttemptId = appAttemptId; } + public AMRMTokenIdentifier(ApplicationAttemptId appAttemptId, + int masterKeyId) { + this(); + this.applicationAttemptId = appAttemptId; + this.keyId = masterKeyId; + } + @Private public ApplicationAttemptId getApplicationAttemptId() { return this.applicationAttemptId; @@ -64,6 +72,7 @@ public void write(DataOutput out) throws IOException { out.writeLong(appId.getClusterTimestamp()); out.writeInt(appId.getId()); out.writeInt(this.applicationAttemptId.getAttemptId()); + out.writeInt(this.keyId); } @Override @@ -75,6 +84,7 @@ public void readFields(DataInput in) throws IOException { ApplicationId.newInstance(clusterTimeStamp, appId); this.applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, attemptId); + this.keyId = in.readInt(); } @Override @@ -92,6 +102,10 @@ public UserGroupInformation getUser() { .toString()); } + public int getKeyId() { + return this.keyId; + } + // TODO: Needed? @InterfaceAudience.Private public static class Renewer extends Token.TrivialRenewer { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 626d47e..3f8a219 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -756,11 +756,9 @@ public void transition(RMAppAttemptImpl appAttempt, } // create AMRMToken - AMRMTokenIdentifier id = - new AMRMTokenIdentifier(appAttempt.applicationAttemptId); appAttempt.amrmToken = - new Token(id, - appAttempt.rmContext.getAMRMTokenSecretManager()); + appAttempt.rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken( + appAttempt.applicationAttemptId); // Add the applicationAttempt to the scheduler and inform the scheduler // whether to transfer the state from previous attempt. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java index 5d21ec0..d2edd95 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java @@ -19,22 +19,25 @@ package org.apache.hadoop.yarn.server.resourcemanager.security; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; +import java.security.SecureRandom; +import java.util.HashSet; +import java.util.Set; import java.util.Timer; import java.util.TimerTask; -import javax.crypto.SecretKey; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.server.security.MasterKeyData; + +import com.google.common.annotations.VisibleForTesting; /** * AMRM-tokens are per ApplicationAttempt. If users redistribute their @@ -49,31 +52,52 @@ private static final Log LOG = LogFactory .getLog(AMRMTokenSecretManager.class); - private SecretKey masterKey; + private int serialNo = new SecureRandom().nextInt(); + private MasterKeyData nextMasterKey; + private MasterKeyData currentMasterKey; + private final Timer timer; private final long rollingInterval; + private final long activationDelay; - private final Map passwords = - new HashMap(); + private final Set appLists = + new HashSet(); /** * Create an {@link AMRMTokenSecretManager} */ public AMRMTokenSecretManager(Configuration conf) { - rollMasterKey(); this.timer = new Timer(); this.rollingInterval = conf .getLong( YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, YarnConfiguration.DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS) * 1000; + // Adding delay = 1.5 * expiry interval makes sure that all active AMs get + // the updated shared-key. + this.activationDelay = + (long) (conf.getLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS) * 1.5); + LOG.info("AMRMTokenKeyRollingInterval: " + this.rollingInterval + + "ms and AMRMTokenKeyActivationDelay: " + this.activationDelay + + "ms"); + if (rollingInterval <= activationDelay * 2) { + throw new IllegalArgumentException( + YarnConfiguration.RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS + + " should be more than 2 X " + + YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS); + } } - public void start() { - this.timer.scheduleAtFixedRate(new MasterKeyRoller(), 0, rollingInterval); + public synchronized void start() { + if (this.currentMasterKey == null) { + this.currentMasterKey = createNewMasterKey(); + } + this.timer.scheduleAtFixedRate(new MasterKeyRoller(), rollingInterval, + rollingInterval); } - public void stop() { + public synchronized void stop() { this.timer.cancel(); } @@ -82,7 +106,7 @@ public synchronized void applicationMasterFinished( if (LOG.isDebugEnabled()) { LOG.debug("Application finished, removing password for " + appAttemptId); } - this.passwords.remove(appAttemptId); + this.appLists.remove(appAttemptId); } private class MasterKeyRoller extends TimerTask { @@ -93,36 +117,51 @@ public void run() { } @Private - public synchronized void setMasterKey(SecretKey masterKey) { - this.masterKey = masterKey; + synchronized void rollMasterKey() { + LOG.info("Rolling master-key for amrm-tokens"); + this.nextMasterKey = createNewMasterKey(); + this.timer.schedule(new NextKeyActivator(), this.activationDelay); } - @Private - public synchronized SecretKey getMasterKey() { - return this.masterKey; + private class NextKeyActivator extends TimerTask { + @Override + public void run() { + activateNextMasterKey(); + } } - @Private - synchronized void rollMasterKey() { - LOG.info("Rolling master-key for amrm-tokens"); - this.masterKey = generateSecret(); + public synchronized void activateNextMasterKey() { + if (LOG.isDebugEnabled()) { + LOG.debug("Activating next master key with id: " + + this.nextMasterKey.getMasterKey().getKeyId()); + } + this.currentMasterKey = this.nextMasterKey; + this.nextMasterKey = null; } - /** - * Create a password for a given {@link AMRMTokenIdentifier}. Used to - * send to the AppicationAttempt which can give it back during authentication. - */ - @Override - public synchronized byte[] createPassword( - AMRMTokenIdentifier identifier) { - ApplicationAttemptId applicationAttemptId = - identifier.getApplicationAttemptId(); - if (LOG.isDebugEnabled()) { - LOG.debug("Creating password for " + applicationAttemptId); + private MasterKeyData createNewMasterKey() { + return new MasterKeyData(serialNo++, generateSecret()); + } + + public synchronized Token createAndGetAMRMToken( + ApplicationAttemptId appAttemptId) { + AMRMTokenIdentifier identifier = + new AMRMTokenIdentifier(appAttemptId, getMasterKey().getMasterKey() + .getKeyId()); + byte[] password = this.createPassword(identifier); + appLists.add(appAttemptId); + return new Token(identifier.getBytes(), password, + identifier.getKind(), new Text()); + } + + // If nextMasterKey is not Null, then return nextMasterKey + // otherwise return currentMasterKey + @VisibleForTesting + public synchronized MasterKeyData getMasterKey() { + if (this.currentMasterKey == null && this.nextMasterKey == null) { + this.currentMasterKey = createNewMasterKey(); } - byte[] password = createPassword(identifier.getBytes(), masterKey); - this.passwords.put(applicationAttemptId, password); - return password; + return nextMasterKey == null ? currentMasterKey : nextMasterKey; } /** @@ -134,8 +173,7 @@ synchronized void rollMasterKey() { if (LOG.isDebugEnabled()) { LOG.debug("Adding password for " + identifier.getApplicationAttemptId()); } - this.passwords.put(identifier.getApplicationAttemptId(), - token.getPassword()); + this.appLists.add(identifier.getApplicationAttemptId()); } /** @@ -143,21 +181,36 @@ synchronized void rollMasterKey() { * Used by RPC layer to validate a remote {@link AMRMTokenIdentifier}. */ @Override - public synchronized byte[] retrievePassword( - AMRMTokenIdentifier identifier) throws InvalidToken { + public synchronized byte[] retrievePassword(AMRMTokenIdentifier identifier) + throws InvalidToken { ApplicationAttemptId applicationAttemptId = identifier.getApplicationAttemptId(); if (LOG.isDebugEnabled()) { LOG.debug("Trying to retrieve password for " + applicationAttemptId); } - byte[] password = this.passwords.get(applicationAttemptId); - if (password == null) { + if (!appLists.contains(applicationAttemptId)) { throw new InvalidToken("Password not found for ApplicationAttempt " + applicationAttemptId); } - return password; + if (identifier.getKeyId() == this.currentMasterKey.getMasterKey() + .getKeyId()) { + LOG.info("AMRMToken password retrieved successfully!!"); + return retrivePasswordInternal(identifier, currentMasterKey); + } else if (nextMasterKey != null + && identifier.getKeyId() == this.nextMasterKey.getMasterKey() + .getKeyId()) { + LOG.info("AMRMToken password retrieved successfully!!"); + return retrivePasswordInternal(identifier, nextMasterKey); + } + throw new InvalidToken("Given AMRMToken for application : " + + applicationAttemptId.toString() + + " seems to have been generated illegally."); } + private byte[] retrivePasswordInternal(AMRMTokenIdentifier identifier, + MasterKeyData masterKey) { + return createPassword(identifier.getBytes(), masterKey.getSecretKey()); + } /** * Creates an empty TokenId to be used for de-serializing an * {@link AMRMTokenIdentifier} by the RPC layer. @@ -167,4 +220,14 @@ public AMRMTokenIdentifier createIdentifier() { return new AMRMTokenIdentifier(); } + @Override + @Private + protected synchronized byte[] createPassword(AMRMTokenIdentifier identifier) { + ApplicationAttemptId applicationAttemptId = + identifier.getApplicationAttemptId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Creating password for " + applicationAttemptId); + } + return createPassword(identifier.getBytes(), getMasterKey().getSecretKey()); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java index 64602bd..4dc1851 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java @@ -23,8 +23,6 @@ import java.util.Arrays; import java.util.Collection; -import javax.crypto.SecretKey; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -50,6 +48,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent; +import org.apache.hadoop.yarn.server.security.MasterKeyData; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; @@ -253,9 +252,9 @@ public void testMasterKeyRollOver() throws Exception { // Simulate a master-key-roll-over AMRMTokenSecretManager appTokenSecretManager = rm.getRMContext().getAMRMTokenSecretManager(); - SecretKey oldKey = appTokenSecretManager.getMasterKey(); + MasterKeyData oldKey = appTokenSecretManager.getMasterKey(); appTokenSecretManager.rollMasterKey(); - SecretKey newKey = appTokenSecretManager.getMasterKey(); + MasterKeyData newKey = appTokenSecretManager.getMasterKey(); Assert.assertFalse("Master key should have changed!", oldKey.equals(newKey));