commit 022056eeb2732d72e332a4651f9131d265a3ad47 Author: Vinod Kumar Vavilapalli Date: Sun Jul 28 12:48:34 2013 -0700 YARN-945 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java index f70b44c..2d2bf7b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java @@ -24,12 +24,17 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; -public class ClientRMProxy extends RMProxy{ +public class ClientRMProxy extends RMProxy { private static final Log LOG = LogFactory.getLog(ClientRMProxy.class); @@ -39,7 +44,24 @@ return createRMProxy(conf, protocol, rmAddress); } - private static InetSocketAddress getRMAddress(Configuration conf, Class protocol) { + private static void setupTokens(InetSocketAddress resourceManagerAddress) + throws IOException { + // It is assumed for now that the only AMRMToken in AM's UGI is for this + // cluster/RM. TODO: Fix later when we have some kind of cluster-ID as + // default service-address + for (Token token : UserGroupInformation + .getCurrentUser().getTokens()) { + if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { + // This token needs to be directly provided to the AMs, so set the + // appropriate service-name. We'll need more infrastructure when we + // need to set it in HA case. + SecurityUtil.setTokenService(token, resourceManagerAddress); + } + } + } + + private static InetSocketAddress getRMAddress(Configuration conf, + Class protocol) throws IOException { if (protocol == ApplicationClientProtocol.class) { return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, @@ -50,10 +72,12 @@ private static InetSocketAddress getRMAddress(Configuration conf, Class proto YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_PORT); } else if (protocol == ApplicationMasterProtocol.class) { - return conf.getSocketAddr( - YarnConfiguration.RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); + InetSocketAddress serviceAddr = + conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); + setupTokens(serviceAddr); + return serviceAddr; } else { String message = "Unsupported protocol found when creating the proxy " + "connection to ResourceManager: " + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java index e4493b5..3b166a8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java @@ -29,23 +29,30 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryProxy; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; + +import com.google.common.annotations.VisibleForTesting; @InterfaceAudience.Public @InterfaceStability.Evolving +@SuppressWarnings("unchecked") public class RMProxy { private static final Log LOG = LogFactory.getLog(RMProxy.class); - @SuppressWarnings("unchecked") public static T createRMProxy(final Configuration conf, final Class protocol, InetSocketAddress rmAddress) throws IOException { RetryPolicy retryPolicy = createRetryPolicy(conf); @@ -54,12 +61,11 @@ return (T) RetryProxy.create(protocol, proxy, retryPolicy); } - @SuppressWarnings("unchecked") - protected static T getProxy(final Configuration conf, + private static T getProxy(final Configuration conf, final Class protocol, final InetSocketAddress rmAddress) throws IOException { - return (T) UserGroupInformation.getCurrentUser().doAs( - new PrivilegedAction() { + return UserGroupInformation.getCurrentUser().doAs( + new PrivilegedAction() { @Override public T run() { @@ -68,6 +74,8 @@ public T run() { }); } + @Private + @VisibleForTesting public static RetryPolicy createRetryPolicy(Configuration conf) { long rmConnectWaitMS = conf.getInt( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java index ef9154f..0f90310 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java @@ -27,7 +27,7 @@ import org.apache.hadoop.yarn.client.RMProxy; import org.apache.hadoop.yarn.conf.YarnConfiguration; -public class ServerRMProxy extends RMProxy{ +public class ServerRMProxy extends RMProxy { private static final Log LOG = LogFactory.getLog(ServerRMProxy.class); @@ -43,8 +43,7 @@ private static InetSocketAddress getRMAddress(Configuration conf, Class proto YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); - } - else { + } else { String message = "Unsupported protocol found when creating the proxy " + "connection to ResourceManager: " + ((protocol != null) ? protocol.getClass().getName() : "null"); @@ -52,4 +51,4 @@ private static InetSocketAddress getRMAddress(Configuration conf, Class proto throw new IllegalStateException(message); } } -} +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 6b8eacd..a1c1a40 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -61,6 +61,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent; import org.apache.hadoop.yarn.util.ConverterUtils; +import com.google.common.annotations.VisibleForTesting; + /** * The launch of the AM itself. */ @@ -224,7 +226,7 @@ private void setupTokens( } // Add AMRMToken - Token amrmToken = application.getAMRMToken(); + Token amrmToken = getAMRMToken(); if (amrmToken != null) { credentials.addToken(amrmToken.getService(), amrmToken); } @@ -232,6 +234,11 @@ private void setupTokens( credentials.writeTokenStorageToStream(dob); container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength())); } + + @VisibleForTesting + protected Token getAMRMToken() { + return application.getAMRMToken(); + } @SuppressWarnings("unchecked") public void run() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index ce6511a..f1cbfa2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.DelegationKey; @@ -43,9 +44,9 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; -import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppStoredEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -376,11 +377,15 @@ public synchronized void removeApplication(ApplicationState appState) { protected abstract void removeApplicationState(ApplicationState appState) throws Exception; + // TODO: This should eventually become cluster-Id + "AM_RM_TOKEN_SERVICE" + public static final Text AM_RM_TOKEN_SERVICE = new Text( + "AM_RM_TOKEN_SERVICE"); + private Credentials getTokensFromAppAttempt(RMAppAttempt appAttempt) { Credentials credentials = new Credentials(); Token appToken = appAttempt.getAMRMToken(); if(appToken != null){ - credentials.addToken(appToken.getService(), appToken); + credentials.addToken(AM_RM_TOKEN_SERVICE, appToken); } Token clientToAMToken = appAttempt.getClientToAMToken(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index ac3fba6..4a877ca 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -20,7 +20,6 @@ import static org.apache.hadoop.yarn.util.StringHelper.pjoin; -import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -41,7 +40,6 @@ import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.ExitUtil; @@ -62,7 +60,6 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; -import org.apache.hadoop.yarn.security.AMRMTokenSelector; import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; @@ -684,15 +681,11 @@ private void recoverAppAttemptTokens(Credentials appAttemptTokens) { appAttemptTokens.getAllTokens()); } - InetSocketAddress serviceAddr = - conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); - AMRMTokenSelector appTokenSelector = new AMRMTokenSelector(); + // Only one AMRMToken is stored per-attempt, so this should be fine. Can't + // use TokenSelector as service may change - think fail-over. this.amrmToken = - appTokenSelector.selectToken( - SecurityUtil.buildTokenService(serviceAddr), - appAttemptTokens.getAllTokens()); + (Token) appAttemptTokens + .getToken(RMStateStore.AM_RM_TOKEN_SERVICE); // For now, no need to populate tokens back to AMRMTokenSecretManager, // because running attempts are rebooted. Later in work-preserve restart, @@ -736,18 +729,9 @@ public void transition(RMAppAttemptImpl appAttempt, // create AMRMToken AMRMTokenIdentifier id = new AMRMTokenIdentifier(appAttempt.applicationAttemptId); - Token amRmToken = + appAttempt.amrmToken = new Token(id, appAttempt.rmContext.getAMRMTokenSecretManager()); - InetSocketAddress serviceAddr = - appAttempt.conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); - // normally the client should set the service after acquiring the - // token, but this token is directly provided to the AMs - SecurityUtil.setTokenService(amRmToken, serviceAddr); - - appAttempt.amrmToken = amRmToken; // Add the application to the scheduler appAttempt.eventHandler.handle( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java index bf6b264..0ea2b5e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java @@ -18,9 +18,15 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import java.net.InetSocketAddress; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; @@ -52,6 +58,17 @@ protected ContainerManagementProtocol getContainerMgrProxy( ContainerId containerId) { return containerManager; } + @Override + protected Token getAMRMToken() { + Token amRmToken = super.getAMRMToken(); + InetSocketAddress serviceAddr = + getConfig().getSocketAddr( + YarnConfiguration.RM_SCHEDULER_ADDRESS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); + SecurityUtil.setTokenService(amRmToken, serviceAddr); + return amRmToken; + } }; } }; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java index af07fb8..1167967 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java @@ -33,7 +33,10 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; @@ -49,6 +52,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -142,6 +146,19 @@ protected void doSecureLogin() throws IOException { protected ApplicationMasterService createApplicationMasterService() { return new ApplicationMasterService(getRMContext(), this.scheduler); } + + @SuppressWarnings("unchecked") + public static Token setupAndReturnAMRMToken( + InetSocketAddress rmBindAddress, + Collection> allTokens) { + for (Token token : allTokens) { + if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { + SecurityUtil.setTokenService(token, rmBindAddress); + return (Token) token; + } + } + return null; + } } @Test @@ -178,8 +195,12 @@ public void testAuthorizedAccess() throws Exception { UserGroupInformation currentUser = UserGroupInformation .createRemoteUser(applicationAttemptId.toString()); Credentials credentials = containerManager.getContainerCredentials(); - currentUser.addCredentials(credentials); - + final InetSocketAddress rmBindAddress = + rm.getApplicationMasterService().getBindAddress(); + Token amRMToken = + MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, + credentials.getAllTokens()); + currentUser.addToken(amRMToken); ApplicationMasterProtocol client = currentUser .doAs(new PrivilegedAction() { @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 27dbe86..64e5cc9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -22,7 +22,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 6cfb387..0f57d1b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -22,6 +22,7 @@ import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; +import java.net.InetSocketAddress; import java.security.PrivilegedAction; import java.util.Collections; import java.util.HashMap; @@ -32,6 +33,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; @@ -47,7 +50,6 @@ import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; -import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -274,7 +276,7 @@ public void testValidateResourceRequest() { public void testValidateResourceBlacklistRequest() throws Exception { MyContainerManager containerManager = new MyContainerManager(); - final MockRM rm = + final MockRMWithAMS rm = new MockRMWithAMS(new YarnConfiguration(), containerManager); rm.start(); @@ -298,13 +300,18 @@ public void testValidateResourceBlacklistRequest() throws Exception { UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString()); Credentials credentials = containerManager.getContainerCredentials(); - currentUser.addCredentials(credentials); - ApplicationMasterProtocol client = currentUser - .doAs(new PrivilegedAction() { + final InetSocketAddress rmBindAddress = + rm.getApplicationMasterService().getBindAddress(); + Token amRMToken = + MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, + credentials.getAllTokens()); + currentUser.addToken(amRMToken); + ApplicationMasterProtocol client = + currentUser.doAs(new PrivilegedAction() { @Override public ApplicationMasterProtocol run() { - return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rm - .getApplicationMasterService().getBindAddress(), conf); + return (ApplicationMasterProtocol) rpc.getProxy( + ApplicationMasterProtocol.class, rmBindAddress, conf); } }); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java index 15c2d74..b0c0488 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.security; +import java.net.InetSocketAddress; import java.security.PrivilegedAction; import java.util.Arrays; import java.util.Collection; @@ -30,6 +31,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; @@ -81,7 +84,7 @@ public TestAMRMTokens(Configuration conf) { public void testTokenExpiry() throws Exception { MyContainerManager containerManager = new MyContainerManager(); - final MockRM rm = + final MockRMWithAMS rm = new MockRMWithAMS(conf, containerManager); rm.start(); @@ -111,8 +114,12 @@ public void testTokenExpiry() throws Exception { UserGroupInformation .createRemoteUser(applicationAttemptId.toString()); Credentials credentials = containerManager.getContainerCredentials(); - currentUser.addCredentials(credentials); - + final InetSocketAddress rmBindAddress = + rm.getApplicationMasterService().getBindAddress(); + Token amRMToken = + MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, + credentials.getAllTokens()); + currentUser.addToken(amRMToken); rmClient = createRMClient(rm, conf, rpc, currentUser); RegisterApplicationMasterRequest request = @@ -164,7 +171,7 @@ public void testTokenExpiry() throws Exception { public void testMasterKeyRollOver() throws Exception { MyContainerManager containerManager = new MyContainerManager(); - final MockRM rm = + final MockRMWithAMS rm = new MockRMWithAMS(conf, containerManager); rm.start(); @@ -194,8 +201,12 @@ public void testMasterKeyRollOver() throws Exception { UserGroupInformation .createRemoteUser(applicationAttemptId.toString()); Credentials credentials = containerManager.getContainerCredentials(); - currentUser.addCredentials(credentials); - + final InetSocketAddress rmBindAddress = + rm.getApplicationMasterService().getBindAddress(); + Token amRMToken = + MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, + credentials.getAllTokens()); + currentUser.addToken(amRMToken); rmClient = createRMClient(rm, conf, rpc, currentUser); RegisterApplicationMasterRequest request =