diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java index 16bbb54..c993b13 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java @@ -126,9 +126,7 @@ public class MRClientService extends AbstractService System .getenv(ApplicationConstants.APPLICATION_CLIENT_SECRET_ENV_NAME); byte[] bytes = Base64.decodeBase64(secretKeyStr); - ClientTokenIdentifier identifier = new ClientTokenIdentifier( - this.appContext.getApplicationID()); - secretManager.setMasterKey(identifier, bytes); + secretManager.setMasterKey(this.appContext.getApplicationID(), bytes); } server = rpc.getServer(MRClientProtocol.class, protocolHandler, address, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMSecretManager.java index 59252e7..e017cfe 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMSecretManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMSecretManager.java @@ -26,8 +26,9 @@ import javax.crypto.SecretKey; import org.apache.commons.codec.binary.Base64; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.util.ConverterUtils; public class ClientToAMSecretManager extends SecretManager { @@ -35,11 +36,11 @@ public class ClientToAMSecretManager extends private static Log LOG = LogFactory.getLog(ClientToAMSecretManager.class); // Per application masterkeys for managing client-tokens - private Map masterKeys = new HashMap(); + private Map masterKeys = + new HashMap(); - public void setMasterKey(ClientTokenIdentifier identifier, byte[] key) { + public void setMasterKey(ApplicationId applicationID, byte[] key) { SecretKey sk = SecretManager.createSecretKey(key); - Text applicationID = identifier.getApplicationID(); this.masterKeys.put(applicationID, sk); if (LOG.isDebugEnabled()) { LOG.debug("Setting master key for " @@ -50,8 +51,7 @@ public class ClientToAMSecretManager extends } } - private void addMasterKey(ClientTokenIdentifier identifier) { - Text applicationID = identifier.getApplicationID(); + private void addMasterKey(ApplicationId applicationID) { this.masterKeys.put(applicationID, generateSecret()); if (LOG.isDebugEnabled()) { LOG.debug("Creating master key for " @@ -62,11 +62,9 @@ public class ClientToAMSecretManager extends } // TODO: Handle the masterKey invalidation. - public synchronized SecretKey getMasterKey( - ClientTokenIdentifier identifier) { - Text applicationID = identifier.getApplicationID(); + public synchronized SecretKey getMasterKey(ApplicationId applicationID) { if (!this.masterKeys.containsKey(applicationID)) { - addMasterKey(identifier); + addMasterKey(applicationID); } return this.masterKeys.get(applicationID); } @@ -74,8 +72,11 @@ public class ClientToAMSecretManager extends @Override public synchronized byte[] createPassword( ClientTokenIdentifier identifier) { + ApplicationId applicationID = ConverterUtils.toApplicationId( + identifier.getApplicationID().toString()); byte[] password = - createPassword(identifier.getBytes(), getMasterKey(identifier)); + createPassword(identifier.getBytes(), + getMasterKey(applicationID)); if (LOG.isDebugEnabled()) { LOG.debug("Password created is " + new String(Base64.encodeBase64(password))); @@ -86,8 +87,8 @@ public class ClientToAMSecretManager extends @Override public byte[] retrievePassword(ClientTokenIdentifier identifier) throws SecretManager.InvalidToken { - byte[] password = - createPassword(identifier.getBytes(), getMasterKey(identifier)); + byte[] password = createPassword(identifier.getBytes(), + getMasterKey(identifier.getApplicationID())); if (LOG.isDebugEnabled()) { LOG.debug("Password retrieved is " + new String(Base64.encodeBase64(password))); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTokenIdentifier.java index 77d97ce..f543ab0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTokenIdentifier.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTokenIdentifier.java @@ -28,36 +28,46 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.util.BuilderUtils; public class ClientTokenIdentifier extends TokenIdentifier { public static final Text KIND_NAME = new Text("YARN_CLIENT_TOKEN"); - private Text appId; + private ApplicationId appId; + private Text clientName; // TODO: Add more information in the tokenID such that it is not // transferrable, more secure etc. - public ClientTokenIdentifier(ApplicationId id) { - this.appId = new Text(Integer.toString(id.getId())); + public ClientTokenIdentifier(ApplicationId id, String clientName) { + this.appId = id; + this.clientName = new Text(clientName); } public ClientTokenIdentifier() { - this.appId = new Text(); + this.clientName = new Text(); } - public Text getApplicationID() { + public ApplicationId getApplicationID() { return appId; } + public Text getClientName() { + return clientName; + } + @Override public void write(DataOutput out) throws IOException { - appId.write(out); + out.writeLong(this.appId.getClusterTimestamp()); + out.writeInt(this.appId.getId()); + clientName.write(out); } @Override public void readFields(DataInput in) throws IOException { - appId.readFields(in); + this.appId = BuilderUtils.newApplicationId(in.readLong(), in.readInt()); + clientName.readFields(in); } @Override @@ -67,10 +77,11 @@ public class ClientTokenIdentifier extends TokenIdentifier { @Override public UserGroupInformation getUser() { - if (appId == null || "".equals(appId.toString())) { + String userName = clientName.toString(); + if (userName.isEmpty()) { return null; } - return UserGroupInformation.createRemoteUser(appId.toString()); + return UserGroupInformation.createRemoteUser(userName); } @InterfaceAudience.Private diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index bc0e61e..7423358 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -27,15 +27,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.ipc.RPCUtil; -import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; -import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -58,19 +55,16 @@ public class RMAppManager implements EventHandler { private LinkedList completedApps = new LinkedList(); private final RMContext rmContext; - private final ClientToAMSecretManager clientToAMSecretManager; private final ApplicationMasterService masterService; private final YarnScheduler scheduler; private final ApplicationACLsManager applicationACLsManager; private Configuration conf; public RMAppManager(RMContext context, - ClientToAMSecretManager clientToAMSecretManager, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationACLsManager applicationACLsManager, Configuration conf) { this.rmContext = context; this.scheduler = scheduler; - this.clientToAMSecretManager = clientToAMSecretManager; this.masterService = masterService; this.applicationACLsManager = applicationACLsManager; this.conf = conf; @@ -230,16 +224,6 @@ public class RMAppManager implements EventHandler { ApplicationId applicationId = submissionContext.getApplicationId(); RMApp application = null; try { - String clientTokenStr = null; - if (UserGroupInformation.isSecurityEnabled()) { - Token clientToken = new - Token( - new ClientTokenIdentifier(applicationId), - this.clientToAMSecretManager); - clientTokenStr = clientToken.encodeToUrlString(); - LOG.debug("Sending client token as " + clientTokenStr); - } - // Sanity checks if (submissionContext.getQueue() == null) { submissionContext.setQueue(YarnConfiguration.DEFAULT_QUEUE_NAME); @@ -259,7 +243,7 @@ public class RMAppManager implements EventHandler { new RMAppImpl(applicationId, rmContext, this.conf, submissionContext.getApplicationName(), submissionContext.getUser(), submissionContext.getQueue(), - submissionContext, clientTokenStr, appStore, this.scheduler, + submissionContext, appStore, this.scheduler, this.masterService, submitTime); // Sanity check - duplicate? diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 592a653..93d74a3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -23,6 +23,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.NodeStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -59,4 +60,6 @@ public interface RMContext { ApplicationTokenSecretManager getApplicationTokenSecretManager(); RMContainerTokenSecretManager getContainerTokenSecretManager(); + + ClientToAMSecretManager getClientToAMSecretManager(); } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 535d841..10bc6d8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.NodeStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store; @@ -54,13 +55,15 @@ public class RMContextImpl implements RMContext { private final DelegationTokenRenewer tokenRenewer; private final ApplicationTokenSecretManager appTokenSecretManager; private final RMContainerTokenSecretManager containerTokenSecretManager; + private final ClientToAMSecretManager clientToAMSecretManager; public RMContextImpl(Store store, Dispatcher rmDispatcher, ContainerAllocationExpirer containerAllocationExpirer, AMLivelinessMonitor amLivelinessMonitor, DelegationTokenRenewer tokenRenewer, ApplicationTokenSecretManager appTokenSecretManager, - RMContainerTokenSecretManager containerTokenSecretManager) { + RMContainerTokenSecretManager containerTokenSecretManager, + ClientToAMSecretManager clientToAMSecretManager) { this.store = store; this.rmDispatcher = rmDispatcher; this.containerAllocationExpirer = containerAllocationExpirer; @@ -68,6 +71,7 @@ public class RMContextImpl implements RMContext { this.tokenRenewer = tokenRenewer; this.appTokenSecretManager = appTokenSecretManager; this.containerTokenSecretManager = containerTokenSecretManager; + this.clientToAMSecretManager = clientToAMSecretManager; } @Override @@ -124,4 +128,9 @@ public class RMContextImpl implements RMContext { public RMContainerTokenSecretManager getContainerTokenSecretManager() { return this.containerTokenSecretManager; } + + @Override + public ClientToAMSecretManager getClientToAMSecretManager() { + return this.clientToAMSecretManager; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 63f4deb..5f83b87 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -162,7 +162,8 @@ public class ResourceManager extends CompositeService implements Recoverable { new RMContextImpl(this.store, this.rmDispatcher, this.containerAllocationExpirer, amLivelinessMonitor, tokenRenewer, this.appTokenSecretManager, - this.containerTokenSecretManager); + this.containerTokenSecretManager, + this.clientToAMSecretManager); addService(nodesListManager); @@ -267,8 +268,7 @@ public class ResourceManager extends CompositeService implements Recoverable { } } protected ApplicationMasterLauncher createAMLauncher() { - return new ApplicationMasterLauncher(this.clientToAMSecretManager, - this.rmContext); + return new ApplicationMasterLauncher(this.rmContext); } private NMLivelinessMonitor createNMLivelinessMonitor() { @@ -285,9 +285,8 @@ public class ResourceManager extends CompositeService implements Recoverable { } protected RMAppManager createRMAppManager() { - return new RMAppManager(this.rmContext, this.clientToAMSecretManager, - this.scheduler, this.masterService, this.applicationACLsManager, - this.conf); + return new RMAppManager(this.rmContext, this.scheduler, this.masterService, + this.applicationACLsManager, this.conf); } @Private diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index aa9d2c2..52bbfec 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -54,8 +54,6 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; -import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; -import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; @@ -76,7 +74,6 @@ public class AMLauncher implements Runnable { private final Configuration conf; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - private final ClientToAMSecretManager clientToAMSecretManager; private final AMLauncherEventType eventType; private final RMContext rmContext; @@ -84,11 +81,9 @@ public class AMLauncher implements Runnable { private final EventHandler handler; public AMLauncher(RMContext rmContext, RMAppAttempt application, - AMLauncherEventType eventType, - ClientToAMSecretManager clientToAMSecretManager, Configuration conf) { + AMLauncherEventType eventType, Configuration conf) { this.application = application; this.conf = conf; - this.clientToAMSecretManager = clientToAMSecretManager; this.eventType = eventType; this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); @@ -237,10 +232,9 @@ public class AMLauncher implements Runnable { container.setContainerTokens( ByteBuffer.wrap(dob.getData(), 0, dob.getLength())); - ClientTokenIdentifier identifier = new ClientTokenIdentifier( - application.getAppAttemptId().getApplicationId()); SecretKey clientSecretKey = - this.clientToAMSecretManager.getMasterKey(identifier); + this.rmContext.getClientToAMSecretManager().getMasterKey( + application.getAppAttemptId().getApplicationId()); String encoded = Base64.encodeBase64URLSafeString(clientSecretKey.getEncoded()); environment.put( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java index 52d201d..a114d80 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java @@ -25,7 +25,6 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.service.AbstractService; @@ -41,17 +40,14 @@ public class ApplicationMasterLauncher extends AbstractService implements private final BlockingQueue masterEvents = new LinkedBlockingQueue(); - private ClientToAMSecretManager clientToAMSecretManager; protected final RMContext context; - public ApplicationMasterLauncher( - ClientToAMSecretManager clientToAMSecretManager, RMContext context) { + public ApplicationMasterLauncher(RMContext context) { super(ApplicationMasterLauncher.class.getName()); this.context = context; this.launcherPool = new ThreadPoolExecutor(10, 10, 1, TimeUnit.HOURS, new LinkedBlockingQueue()); this.launcherHandlingThread = new LauncherThread(); - this.clientToAMSecretManager = clientToAMSecretManager; } public void start() { @@ -62,8 +58,7 @@ public class ApplicationMasterLauncher extends AbstractService implements protected Runnable createRunnableLauncher(RMAppAttempt application, AMLauncherEventType event) { Runnable launcher = - new AMLauncher(context, application, event, clientToAMSecretManager, - getConfig()); + new AMLauncher(context, application, event, getConfig()); return launcher; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index c516e8a..f1d4b3b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -76,7 +76,6 @@ public class RMAppImpl implements RMApp { private final String queue; private final String name; private final ApplicationSubmissionContext submissionContext; - private final String clientTokenStr; private final ApplicationStore appStore; private final Dispatcher dispatcher; private final YarnScheduler scheduler; @@ -175,7 +174,7 @@ public class RMAppImpl implements RMApp { public RMAppImpl(ApplicationId applicationId, RMContext rmContext, Configuration config, String name, String user, String queue, - ApplicationSubmissionContext submissionContext, String clientTokenStr, + ApplicationSubmissionContext submissionContext, ApplicationStore appStore, YarnScheduler scheduler, ApplicationMasterService masterService, long submitTime) { @@ -189,7 +188,6 @@ public class RMAppImpl implements RMApp { this.user = user; this.queue = queue; this.submissionContext = submissionContext; - this.clientTokenStr = clientTokenStr; this.appStore = appStore; this.scheduler = scheduler; this.masterService = masterService; @@ -468,8 +466,7 @@ public class RMAppImpl implements RMApp { appAttemptId.setAttemptId(attempts.size() + 1); RMAppAttempt attempt = new RMAppAttemptImpl(appAttemptId, - clientTokenStr, rmContext, scheduler, masterService, - submissionContext, conf); + rmContext, scheduler, masterService, submissionContext, conf); attempts.put(appAttemptId, attempt); currentAttempt = attempt; handler.handle( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 6346bb3..c5866a2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; import static org.apache.hadoop.yarn.util.StringHelper.pjoin; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -37,6 +38,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -52,6 +55,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; @@ -110,7 +114,6 @@ public class RMAppAttemptImpl implements RMAppAttempt { private final WriteLock writeLock; private final ApplicationAttemptId applicationAttemptId; - private final String clientToken; private final ApplicationSubmissionContext submissionContext; //nodes on while this attempt's containers ran @@ -265,7 +268,7 @@ public class RMAppAttemptImpl implements RMAppAttempt { .installTopology(); public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, - String clientToken, RMContext rmContext, YarnScheduler scheduler, + RMContext rmContext, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationSubmissionContext submissionContext, Configuration conf) { @@ -277,7 +280,6 @@ public class RMAppAttemptImpl implements RMAppAttempt { this.submissionContext = submissionContext; this.scheduler = scheduler; this.masterService = masterService; - this.clientToken = clientToken; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); @@ -403,7 +405,22 @@ public class RMAppAttemptImpl implements RMAppAttempt { @Override public String getClientToken() { - return this.clientToken; + String tokenStr = null; + if (UserGroupInformation.isSecurityEnabled()) { + try { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + ClientTokenIdentifier tokenId = + new ClientTokenIdentifier(applicationAttemptId.getApplicationId(), + ugi.getUserName()); + Token token = + new Token(tokenId, + this.rmContext.getClientToAMSecretManager()); + tokenStr = token.encodeToUrlString(); + } catch (IOException e) { + LOG.error("Error generating client-to-AM token", e); + } + } + return tokenStr; } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 465be8c..9ca0f11 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -275,8 +275,7 @@ public class MockRM extends ResourceManager { @Override protected ApplicationMasterLauncher createAMLauncher() { - return new ApplicationMasterLauncher(this.clientToAMSecretManager, - getRMContext()) { + return new ApplicationMasterLauncher(getRMContext()) { @Override public void start() { // override to not start rpc handler diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 0fabfcf..5b718a5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -92,7 +92,8 @@ public class TestAppManager{ AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor( rmDispatcher); return new RMContextImpl(new MemStore(), rmDispatcher, - containerAllocationExpirer, amLivelinessMonitor, null, null, null) { + containerAllocationExpirer, amLivelinessMonitor, null, null, null, + null) { @Override public ConcurrentMap getRMApps() { return map; @@ -132,15 +133,14 @@ public class TestAppManager{ public class TestRMAppManager extends RMAppManager { public TestRMAppManager(RMContext context, Configuration conf) { - super(context, null, null, null, new ApplicationACLsManager(conf), conf); + super(context, null, null, new ApplicationACLsManager(conf), conf); setCompletedAppsMax(YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS); } public TestRMAppManager(RMContext context, - ClientToAMSecretManager clientToAMSecretManager, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationACLsManager applicationACLsManager, Configuration conf) { - super(context, clientToAMSecretManager, scheduler, masterService, + super(context, scheduler, masterService, applicationACLsManager, conf); setCompletedAppsMax(YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS); } @@ -339,8 +339,7 @@ public class TestAppManager{ ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); TestRMAppManager appMonitor = new TestRMAppManager(rmContext, - new ClientToAMSecretManager(), scheduler, masterService, - new ApplicationACLsManager(conf), conf); + scheduler, masterService, new ApplicationACLsManager(conf), conf); ApplicationId appID = MockApps.newAppID(1); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -387,8 +386,7 @@ public class TestAppManager{ ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); TestRMAppManager appMonitor = new TestRMAppManager(rmContext, - new ClientToAMSecretManager(), scheduler, masterService, - new ApplicationACLsManager(conf), conf); + scheduler, masterService, new ApplicationACLsManager(conf), conf); ApplicationId appID = MockApps.newAppID(10); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -435,8 +433,7 @@ public class TestAppManager{ ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); TestRMAppManager appMonitor = new TestRMAppManager(rmContext, - new ClientToAMSecretManager(), scheduler, masterService, - new ApplicationACLsManager(conf), conf); + scheduler, masterService, new ApplicationACLsManager(conf), conf); ApplicationId appID = MockApps.newAppID(0); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 5948ffe..135312d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -121,13 +121,11 @@ public class TestApplicationMasterLauncher { @Override protected ApplicationMasterLauncher createAMLauncher() { - return new ApplicationMasterLauncher(super.clientToAMSecretManager, - getRMContext()) { + return new ApplicationMasterLauncher(getRMContext()) { @Override protected Runnable createRunnableLauncher(RMAppAttempt application, AMLauncherEventType event) { - return new AMLauncher(context, application, event, - clientToAMSecretManager, getConfig()) { + return new AMLauncher(context, application, event, getConfig()) { @Override protected ContainerManager getContainerMgrProxy( ContainerId containerId) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index b82cb1e..13d6789 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -36,9 +36,7 @@ import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.MockApps; @@ -256,7 +254,7 @@ public class TestClientRMService { ApplicationsStore stateStore = mock(ApplicationsStore.class); when(rmContext.getApplicationsStore()).thenReturn(stateStore); mockRMContext(yarnScheduler, rmContext); - RMAppManager appManager = new RMAppManager(rmContext, null, yarnScheduler, + RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration()); final ApplicationId appId1 = getApplicationId(100); @@ -371,7 +369,7 @@ public class TestClientRMService { private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, ApplicationId applicationId3, YarnConfiguration config, String queueName) { return new RMAppImpl(applicationId3, rmContext, config, null, null, - queueName, null, null, null, yarnScheduler, null, System + queueName, null, null, yarnScheduler, null, System .currentTimeMillis()); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index 845e13e..ad2b416 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -83,7 +83,7 @@ public class TestRMNodeTransitions { rmContext = new RMContextImpl(new MemStore(), rmDispatcher, null, null, - mock(DelegationTokenRenewer.class), null, null); + mock(DelegationTokenRenewer.class), null, null, null); scheduler = mock(YarnScheduler.class); doAnswer( new Answer() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java index 75f01d3..c0443f9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java @@ -71,7 +71,7 @@ public class TestNMExpiry { // Dispatcher that processes events inline Dispatcher dispatcher = new InlineDispatcher(); RMContext context = new RMContextImpl(new MemStore(), dispatcher, null, - null, null, null, null); + null, null, null, null, null); dispatcher.register(SchedulerEventType.class, new InlineDispatcher.EmptyEventHandler()); dispatcher.register(RMNodeEventType.class, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java index d88d05e..96ec22c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java @@ -66,7 +66,8 @@ public class TestRMNMRPCResponseId { } }); RMContext context = - new RMContextImpl(new MemStore(), dispatcher, null, null, null, null, null); + new RMContextImpl(new MemStore(), dispatcher, null, null, null, null, + null, null); dispatcher.register(RMNodeEventType.class, new ResourceManager.NodeEventDispatcher(context)); NodesListManager nodesListManager = new NodesListManager(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 3db98fe..cebed4d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -123,7 +124,8 @@ public class TestRMAppTransitions { new RMContextImpl(new MemStore(), rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, null, new ApplicationTokenSecretManager(conf), - new RMContainerTokenSecretManager(conf)); + new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager()); rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher(this.rmContext)); @@ -143,7 +145,6 @@ public class TestRMAppTransitions { // ensure max retries set to known value conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, maxRetries); ApplicationSubmissionContext submissionContext = null; - String clientTokenStr = "bogusstring"; ApplicationStore appStore = mock(ApplicationStore.class); YarnScheduler scheduler = mock(YarnScheduler.class); ApplicationMasterService masterService = @@ -151,7 +152,7 @@ public class TestRMAppTransitions { RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, - queue, submissionContext, clientTokenStr, + queue, submissionContext, appStore, scheduler, masterService, System.currentTimeMillis()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 0bc1ed4..fd05395 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -158,7 +159,8 @@ public class TestRMAppAttemptTransitions { new RMContextImpl(new MemStore(), rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, null, new ApplicationTokenSecretManager(conf), - new RMContainerTokenSecretManager(conf)); + new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager()); scheduler = mock(YarnScheduler.class); masterService = mock(ApplicationMasterService.class); @@ -197,7 +199,7 @@ public class TestRMAppAttemptTransitions { application = mock(RMApp.class); applicationAttempt = - new RMAppAttemptImpl(applicationAttemptId, null, rmContext, scheduler, + new RMAppAttemptImpl(applicationAttemptId, rmContext, scheduler, masterService, submissionContext, new Configuration()); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 18049cc..6be7d5d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.Application; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -263,7 +264,8 @@ public class TestCapacityScheduler { setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf))); + null, new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager())); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); conf.setCapacity(A, 80f); @@ -360,7 +362,7 @@ public class TestCapacityScheduler { conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, - new RMContainerTokenSecretManager(conf))); + new RMContainerTokenSecretManager(conf), new ClientToAMSecretManager())); } @Test @@ -371,7 +373,7 @@ public class TestCapacityScheduler { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null, null, - new RMContainerTokenSecretManager(csConf))); + new RMContainerTokenSecretManager(csConf), new ClientToAMSecretManager())); RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1); RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2); @@ -396,7 +398,7 @@ public class TestCapacityScheduler { setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, - new RMContainerTokenSecretManager(conf))); + new RMContainerTokenSecretManager(conf), new ClientToAMSecretManager())); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); // Add a new queue b4 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 88e3e37..b5492bc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -23,6 +23,7 @@ import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.junit.Test; @@ -43,7 +44,8 @@ public class TestQueueParsing { CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.setConf(conf); capacityScheduler.reinitialize(conf, new RMContextImpl(null, null, null, - null, null, null, new RMContainerTokenSecretManager(conf))); + null, null, null, new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager())); CSQueue a = capacityScheduler.getQueue("a"); Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 6fa7723..3d59014 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; @@ -84,7 +85,8 @@ public class TestUtils { RMContext rmContext = new RMContextImpl(null, nullDispatcher, cae, null, null, new ApplicationTokenSecretManager(conf), - new RMContainerTokenSecretManager(conf)); + new RMContainerTokenSecretManager(conf), + new ClientToAMSecretManager()); return rmContext; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 33819b7..81ff269 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -91,7 +91,7 @@ public class TestFifoScheduler { public void testAppAttemptMetrics() throws Exception { AsyncDispatcher dispatcher = new InlineDispatcher(); RMContext rmContext = - new RMContextImpl(null, dispatcher, null, null, null, null, null); + new RMContextImpl(null, dispatcher, null, null, null, null, null, null); FifoScheduler schedular = new FifoScheduler(); schedular.reinitialize(new Configuration(), rmContext); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index f82d2d0..e54af83 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; @@ -159,7 +160,8 @@ public class TestRMWebApp { for (RMNode node : deactivatedNodes) { deactivatedNodesMap.put(node.getHostName(), node); } - return new RMContextImpl(new MemStore(), null, null, null, null, null, null) { + return new RMContextImpl(new MemStore(), null, null, null, null, null, + null, null) { @Override public ConcurrentMap getRMApps() { return applicationsMaps; @@ -200,7 +202,7 @@ public class TestRMWebApp { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, - new RMContainerTokenSecretManager(conf))); + new RMContainerTokenSecretManager(conf), new ClientToAMSecretManager())); return cs; }