diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index fae7074..706cb1c 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -762,11 +762,9 @@ private static ContainerLaunchContext createCommonContainerLaunchContext( // Construct the actual Container // The null fields are per-container and will be constructed for each // container separately. - ContainerLaunchContext container = BuilderUtils - .newContainerLaunchContext(conf - .get(MRJobConfig.USER_NAME), localResources, - environment, null, serviceData, taskCredentialsBuffer, - applicationACLs); + ContainerLaunchContext container = + BuilderUtils.newContainerLaunchContext(localResources, environment, + null, serviceData, taskCredentialsBuffer, applicationACLs); return container; } @@ -807,11 +805,11 @@ static ContainerLaunchContext createContainerLaunchContext( } // Construct the actual Container - ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext( - commonContainerSpec.getUser(), - commonContainerSpec.getLocalResources(), myEnv, commands, - myServiceData, commonContainerSpec.getContainerTokens().duplicate(), - applicationACLs); + ContainerLaunchContext container = + BuilderUtils.newContainerLaunchContext( + commonContainerSpec.getLocalResources(), myEnv, commands, + myServiceData, commonContainerSpec.getContainerTokens().duplicate(), + applicationACLs); return container; } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index 4ef4d8d..321df09 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -519,7 +519,7 @@ public void handle(ContainerAllocatorEvent event) { cId.setId(containerCount++); NodeId nodeId = BuilderUtils.newNodeId(NM_HOST, NM_PORT); Container container = BuilderUtils.newContainer(cId, nodeId, - NM_HOST + ":" + NM_HTTP_PORT, null, null, null); + NM_HOST + ":" + NM_HTTP_PORT, null, null, "user", null); JobID id = TypeConverter.fromYarn(applicationId); JobId jobId = TypeConverter.toYarn(id); getContext().getEventHandler().handle(new JobHistoryEvent(jobId, diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java index efb8b7a..ab8e5a1 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java @@ -243,7 +243,7 @@ public AllocateResponse allocate(AllocateRequest request) .newContainer(containerId, BuilderUtils.newNodeId("host" + containerId.getId(), 2345), "host" + containerId.getId() + ":5678", req - .getCapability(), req.getPriority(), null)); + .getCapability(), req.getPriority(), "user", null)); } } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 241258a..518971b 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -467,10 +467,9 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB)); // Setup ContainerLaunchContext for AM container - ContainerLaunchContext amContainer = BuilderUtils - .newContainerLaunchContext(UserGroupInformation - .getCurrentUser().getShortUserName(), localResources, - environment, vargsFinal, null, securityTokens, acls); + ContainerLaunchContext amContainer = + BuilderUtils.newContainerLaunchContext(localResources, environment, + vargsFinal, null, securityTokens, acls); // Set up the ApplicationSubmissionContext ApplicationSubmissionContext appContext = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationStateData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationStateData.java index 9b1e14a..b45edb8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationStateData.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationStateData.java @@ -40,8 +40,24 @@ @Public @Unstable public void setSubmitTime(long submitTime); + + /** + * Get the user to whom the container has been allocated. + * @return the user to whom the container has been allocated + */ + @Public + @Unstable + String getUser(); /** + * Set the user to whom the container has been allocated + * @param user user to whom the container has been allocated + */ + @Public + @Unstable + void setUser(String user); + + /** * The {@link ApplicationSubmissionContext} for the application * {@link ApplicationId} can be obtained from the this * @return ApplicationSubmissionContext diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java index 9478d34..c90121e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java @@ -113,6 +113,22 @@ void setResource(Resource resource); /** + * Get the user to whom the container has been allocated. + * @return the user to whom the container has been allocated + */ + @Public + @Stable + String getUser(); + + /** + * Set the user to whom the container has been allocated + * @param user user to whom the container has been allocated + */ + @Public + @Stable + void setUser(String user); + + /** * Get the Priority at which the Container was * allocated. * @return Priority at which the Container was diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java index 36cfdfb..d73ec61 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java @@ -51,21 +51,6 @@ @Public @Stable public interface ContainerLaunchContext { - /** - * Get the user to whom the container has been allocated. - * @return the user to whom the container has been allocated - */ - @Public - @Stable - String getUser(); - - /** - * Set the user to whom the container has been allocated - * @param user user to whom the container has been allocated - */ - @Public - @Stable - void setUser(String user); /** * Get security tokens (if security is enabled). diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationStateDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationStateDataPBImpl.java index dced423..6f2ce60 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationStateDataPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationStateDataPBImpl.java @@ -91,6 +91,21 @@ public void setSubmitTime(long submitTime) { } @Override + public String getUser() { + ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasUser()) { + return null; + } + return (p.getUser()); + } + + @Override + public void setUser(String user) { + maybeInitBuilder(); + builder.setUser(user); + } + + @Override public ApplicationSubmissionContext getApplicationSubmissionContext() { ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; if(applicationSubmissionContext != null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java index 6a26508..77e7153 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java @@ -142,25 +142,6 @@ private void addCommandsToProto() { } @Override - public String getUser() { - ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasUser()) { - return null; - } - return (p.getUser()); - } - - @Override - public void setUser(String user) { - maybeInitBuilder(); - if (user == null) { - builder.clearUser(); - return; - } - builder.setUser((user)); - } - - @Override public Map getLocalResources() { initLocalResources(); return this.localResources; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index dd6941f..13e9ac0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ProtoBase; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; @@ -210,6 +211,25 @@ public void setPriority(Priority priority) { } @Override + public String getUser() { + ContainerProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasUser()) { + return null; + } + return (p.getUser()); + } + + @Override + public void setUser(String user) { + maybeInitBuilder(); + if (user == null) { + builder.clearUser(); + return; + } + builder.setUser((user)); + } + + @Override public ContainerToken getContainerToken() { ContainerProtoOrBuilder p = viaProto ? proto : builder; if (this.containerToken != null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index c865c5d..a511a2c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -67,7 +67,8 @@ message ContainerProto { optional string node_http_address = 3; optional ResourceProto resource = 4; optional PriorityProto priority = 5; - optional hadoop.common.TokenProto container_token = 6; + optional string user = 6; + optional hadoop.common.TokenProto container_token = 7; } enum YarnApplicationStateProto { @@ -265,13 +266,12 @@ message QueueUserACLInfoProto { //////////////////////////////////////////////////////////////////////// message ContainerLaunchContextProto { - optional string user = 1; - repeated StringLocalResourceMapProto localResources = 2; - optional bytes container_tokens = 3; - repeated StringBytesMapProto service_data = 4; - repeated StringStringMapProto environment = 5; - repeated string command = 6; - repeated ApplicationACLMapProto application_ACLs = 7; + repeated StringLocalResourceMapProto localResources = 1; + optional bytes container_tokens = 2; + repeated StringBytesMapProto service_data = 3; + repeated StringStringMapProto environment = 4; + repeated string command = 5; + repeated ApplicationACLMapProto application_ACLs = 6; } message ContainerStatusProto { @@ -316,7 +316,8 @@ message StringBytesMapProto { //////////////////////////////////////////////////////////////////////// message ApplicationStateDataProto { optional int64 submit_time = 1; - optional ApplicationSubmissionContextProto application_submission_context = 2; + optional string user = 2; + optional ApplicationSubmissionContextProto application_submission_context = 3; } message ApplicationAttemptStateDataProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 965a8aa..0c1a807 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -38,7 +38,6 @@ import org.apache.commons.cli.ParseException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -47,13 +46,11 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ContainerExitStatus; import org.apache.hadoop.yarn.api.ContainerManager; - import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; - import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -680,11 +677,6 @@ public void run() { ContainerLaunchContext ctx = Records .newRecord(ContainerLaunchContext.class); - String jobUserName = System.getenv(ApplicationConstants.Environment.USER - .key()); - ctx.setUser(jobUserName); - LOG.info("Setting user in ContainerLaunchContext to: " + jobUserName); - // Set the environment ctx.setEnvironment(shellEnv); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClientAsync.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClientAsync.java index bbb8ad1..b1cff9d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClientAsync.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClientAsync.java @@ -18,9 +18,9 @@ package org.apache.hadoop.yarn.client; -import static org.mockito.Mockito.anyFloat; -import static org.mockito.Mockito.anyInt; -import static org.mockito.Mockito.anyString; +import static org.mockito.Matchers.anyFloat; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -56,7 +56,7 @@ public void testAMRMClientAsync() throws Exception { BuilderUtils.newContainerId(0, 0, 0, 0), ContainerState.COMPLETE, "", 0)); List allocated1 = Arrays.asList( - BuilderUtils.newContainer(null, null, null, null, null, null)); + BuilderUtils.newContainer(null, null, null, null, null, null, null)); final AllocateResponse response1 = createAllocateResponse( new ArrayList(), allocated1); final AllocateResponse response2 = createAllocateResponse(completed1, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java index c7502c1..0aedf9b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java @@ -237,14 +237,15 @@ public static ContainerStatus newContainerStatus(ContainerId containerId, return containerStatus; } - public static Container newContainer(ContainerId containerId, - NodeId nodeId, String nodeHttpAddress, - Resource resource, Priority priority, ContainerToken containerToken) { + public static Container newContainer(ContainerId containerId, NodeId nodeId, + String nodeHttpAddress, Resource resource, Priority priority, + String user, ContainerToken containerToken) { Container container = recordFactory.newRecordInstance(Container.class); container.setId(containerId); container.setNodeId(nodeId); container.setNodeHttpAddress(nodeHttpAddress); container.setResource(resource); + container.setUser(user); container.setPriority(priority); container.setContainerToken(containerToken); return container; @@ -284,13 +285,12 @@ public static ContainerToken newContainerToken(NodeId nodeId, } public static ContainerLaunchContext newContainerLaunchContext( - String user, Map localResources, + Map localResources, Map environment, List commands, Map serviceData, ByteBuffer containerTokens, Map acls) { ContainerLaunchContext container = recordFactory .newRecordInstance(ContainerLaunchContext.class); - container.setUser(user); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java index 295a38c..ad40af4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java @@ -90,7 +90,6 @@ private void testRPCTimeout(String rpcClass) throws Exception { server.getListenerAddress(), conf); ContainerLaunchContext containerLaunchContext = recordFactory .newRecordInstance(ContainerLaunchContext.class); - containerLaunchContext.setUser("dummy-user"); ContainerId containerId = recordFactory .newRecordInstance(ContainerId.class); ApplicationId applicationId = recordFactory @@ -105,7 +104,7 @@ private void testRPCTimeout(String rpcClass) throws Exception { containerId.setId(100); Container container = BuilderUtils.newContainer(containerId, null, null, recordFactory - .newRecordInstance(Resource.class), null, null); + .newRecordInstance(Resource.class), null, "dummy-user", null); StartContainerRequest scRequest = recordFactory .newRecordInstance(StartContainerRequest.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java index 7d941e9..ca25f41 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java @@ -113,7 +113,6 @@ private void test(String rpcClass) throws Exception { NetUtils.getConnectAddress(server), conf); ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - containerLaunchContext.setUser("dummy-user"); ContainerId containerId = recordFactory.newRecordInstance(ContainerId.class); ApplicationId applicationId = @@ -128,9 +127,7 @@ private void test(String rpcClass) throws Exception { containerId.setId(100); Container mockContainer = BuilderUtils.newContainer(containerId, null, null, recordFactory - .newRecordInstance(Resource.class), null, null); -// containerLaunchContext.env = new HashMap(); -// containerLaunchContext.command = new ArrayList(); + .newRecordInstance(Resource.class), null, "dummy-user", null); StartContainerRequest scRequest = recordFactory.newRecordInstance(StartContainerRequest.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java index 12020c7..b1aec1b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java @@ -21,7 +21,9 @@ import java.util.List; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationStateDataPBImpl; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; @@ -37,6 +39,15 @@ private static final RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(null); + public static ApplicationStateDataPBImpl newApplicationStateData( + long submitTime, String user, ApplicationSubmissionContext context) { + ApplicationStateDataPBImpl appStateData = new ApplicationStateDataPBImpl(); + appStateData.setSubmitTime(submitTime); + appStateData.setUser(user); + appStateData.setApplicationSubmissionContext(context); + return appStateData; + } + public static NodeHeartbeatResponse newNodeHeartbeatResponse(int responseId, NodeAction action, List containersToCleanUp, List applicationsToCleanUp, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 8fc8a3e..cfc1bef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -23,8 +23,6 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Set; @@ -344,12 +342,13 @@ private void authorizeRequest(String containerIDStr, + containerIDStr); } else { + String userName = container.getUser(); + // Is the container coming in with correct user-name? - if (!tokenId.getApplicationSubmitter().equals(launchContext.getUser())) { + if (!tokenId.getApplicationSubmitter().equals(userName)) { unauthorized = true; messageBuilder.append("\n Expected user-name " - + tokenId.getApplicationSubmitter() + " but found " - + launchContext.getUser()); + + tokenId.getApplicationSubmitter() + " but found " + userName); } // Is the container being relaunched? Or RPC layer let startCall with @@ -402,8 +401,8 @@ public StartContainerResponse startContainer(StartContainerRequest request) UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr); authorizeRequest(containerIDStr, launchContext, lauchContainer, remoteUgi); - LOG.info("Start request for " + containerIDStr + " by user " - + launchContext.getUser()); + String user = lauchContainer.getUser(); + LOG.info("Start request for " + containerIDStr + " by user " + user); // //////////// Parse credentials ByteBuffer tokens = launchContext.getContainerTokens(); @@ -427,22 +426,21 @@ public StartContainerResponse startContainer(StartContainerRequest request) // //////////// End of parsing credentials Container container = new ContainerImpl(getConfig(), this.dispatcher, - launchContext, lauchContainer, credentials, metrics); + launchContext, user, lauchContainer, credentials, metrics); ApplicationId applicationID = containerID.getApplicationAttemptId().getApplicationId(); if (context.getContainers().putIfAbsent(containerID, container) != null) { - NMAuditLogger.logFailure(launchContext.getUser(), - AuditConstants.START_CONTAINER, "ContainerManagerImpl", - "Container already running on this node!", - applicationID, containerID); + NMAuditLogger.logFailure(user, AuditConstants.START_CONTAINER, + "ContainerManagerImpl", "Container already running on this node!", + applicationID, containerID); throw RPCUtil.getRemoteException("Container " + containerIDStr + " already is running on this node!!"); } // Create the application Application application = - new ApplicationImpl(dispatcher, this.aclsManager, - launchContext.getUser(), applicationID, credentials, context); + new ApplicationImpl(dispatcher, this.aclsManager, user, + applicationID, credentials, context); if (null == context.getApplications().putIfAbsent(applicationID, application)) { LOG.info("Creating a new application reference for app " @@ -462,9 +460,8 @@ public StartContainerResponse startContainer(StartContainerRequest request) tokenId); } - NMAuditLogger.logSuccess(launchContext.getUser(), - AuditConstants.START_CONTAINER, "ContainerManageImpl", - applicationID, containerID); + NMAuditLogger.logSuccess(user, AuditConstants.START_CONTAINER, + "ContainerManageImpl", applicationID, containerID); StartContainerResponse response = recordFactory.newRecordInstance(StartContainerResponse.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 961c454..e67a74d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -46,8 +46,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger; import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent; @@ -78,6 +76,7 @@ private final Credentials credentials; private final NodeManagerMetrics metrics; private final ContainerLaunchContext launchContext; + private final String user; private final org.apache.hadoop.yarn.api.records.Container container; private int exitCode = ContainerExitStatus.INVALID; private final StringBuilder diagnostics; @@ -86,7 +85,6 @@ private final Configuration daemonConf; private static final Log LOG = LogFactory.getLog(Container.class); - private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); private final Map> pendingResources = new HashMap>(); private final Map> localizedResources = @@ -99,12 +97,13 @@ new ArrayList(); public ContainerImpl(Configuration conf, - Dispatcher dispatcher, ContainerLaunchContext launchContext, + Dispatcher dispatcher, ContainerLaunchContext launchContext, String user, org.apache.hadoop.yarn.api.records.Container container, Credentials creds, NodeManagerMetrics metrics) { this.daemonConf = conf; this.dispatcher = dispatcher; this.launchContext = launchContext; + this.user = user; this.container = container; this.diagnostics = new StringBuilder(); this.credentials = creds; @@ -326,7 +325,7 @@ public ContainerId getContainerID() { public String getUser() { this.readLock.lock(); try { - return this.launchContext.getUser(); + return this.user; } finally { this.readLock.unlock(); } @@ -397,7 +396,7 @@ public Resource getResource() { @SuppressWarnings({"fallthrough", "unchecked"}) private void finished() { ContainerId containerID = this.container.getId(); - String user = this.launchContext.getUser(); + String user = this.getUser(); switch (getContainerState()) { case EXITED_WITH_SUCCESS: metrics.endRunningContainer(); @@ -501,7 +500,7 @@ public ContainerState transition(ContainerImpl container, for (Map.Entry service : csd.entrySet()) { container.dispatcher.getEventHandler().handle( new AuxServicesEvent(AuxServicesEventType.APPLICATION_INIT, - ctxt.getUser(), container.container.getId() + container.user, container.container.getId() .getApplicationAttemptId().getApplicationId(), service.getKey().toString(), service.getValue())); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 876a57f..1ef57af 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -22,8 +22,8 @@ import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; import java.io.DataOutputStream; -import java.io.IOException; import java.io.File; +import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; import java.util.ArrayList; @@ -53,10 +53,10 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; -import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.DelayedProcessKiller; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent; @@ -119,7 +119,7 @@ public Integer call() { container.getLocalizedResources(); ContainerId containerID = container.getContainerID(); String containerIdStr = ConverterUtils.toString(containerID); - final String user = launchContext.getUser(); + final String user = container.getUser(); final List command = launchContext.getCommands(); int ret = -1; @@ -333,7 +333,7 @@ public void cleanupContainer() throws IOException { // kill process if (processId != null) { - String user = container.getLaunchContext().getUser(); + String user = container.getUser(); LOG.debug("Sending signal to pid " + processId + " as user " + user + " for container " + containerIdStr); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java index 44328db..220350a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java @@ -24,7 +24,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager; @@ -35,7 +34,7 @@ private static final Log LOG = LogFactory .getLog(TestContainerManagerWithLCE.class); - public TestContainerManagerWithLCE() throws UnsupportedFileSystemException { + public TestContainerManagerWithLCE() throws IOException { super(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java index 292d00f..2f9e908 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.server.nodemanager; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.io.File; import java.io.IOException; @@ -44,7 +47,6 @@ import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.junit.Test; -import static org.mockito.Mockito.*; public class TestEventFlow { @@ -121,10 +123,10 @@ protected void startStatusUpdater() { applicationAttemptId.setAttemptId(0); cID.setApplicationAttemptId(applicationAttemptId); Container mockContainer = mock(Container.class); + when(mockContainer.getUser()).thenReturn("testing"); when(mockContainer.getId()).thenReturn(cID); when(mockContainer.getResource()).thenReturn(recordFactory .newRecordInstance(Resource.class)); - launchContext.setUser("testing"); StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class); request.setContainerLaunchContext(launchContext); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java index 1436193..5fdaeda 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java @@ -18,7 +18,14 @@ package org.apache.hadoop.yarn.server.nodemanager; -import static org.mockito.Mockito.*; +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.File; import java.io.IOException; @@ -99,11 +106,6 @@ public void testClearLocalDirWhenNodeReboot() throws IOException { Records.newRecord(ContainerLaunchContext.class); // Construct the Container-id ContainerId cId = createContainerId(); - org.apache.hadoop.yarn.api.records.Container mockContainer = - mock(org.apache.hadoop.yarn.api.records.Container.class); - when(mockContainer.getId()).thenReturn(cId); - - containerLaunchContext.setUser(user); URL localResourceUri = ConverterUtils.getYarnUrlFromPath(localFS @@ -121,15 +123,18 @@ public void testClearLocalDirWhenNodeReboot() throws IOException { new HashMap(); localResources.put(destinationFile, localResource); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = new ArrayList(); containerLaunchContext.setCommands(commands); Resource resource = Records.newRecord(Resource.class); resource.setMemory(1024); - when(mockContainer.getResource()).thenReturn(resource); StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); + org.apache.hadoop.yarn.api.records.Container mockContainer = + mock(org.apache.hadoop.yarn.api.records.Container.class); + when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getUser()).thenReturn(user); + when(mockContainer.getResource()).thenReturn(resource); startRequest.setContainer(mockContainer); containerManager.startContainer(startRequest); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index 1efe80d..e354ef8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -161,12 +161,6 @@ private void startContainers(NodeManager nm) throws IOException { ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - Container mockContainer = mock(Container.class); - // Construct the Container-id - ContainerId cId = createContainerId(); - when(mockContainer.getId()).thenReturn(cId); - - containerLaunchContext.setUser(user); URL localResourceUri = ConverterUtils.getYarnUrlFromPath(localFS @@ -183,16 +177,21 @@ private void startContainers(NodeManager nm) throws IOException { new HashMap(); localResources.put(destinationFile, localResource); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = new ArrayList(); commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); Resource resource = BuilderUtils.newResource(1024, 1); - when(mockContainer.getResource()).thenReturn(resource); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); + Container mockContainer = mock(Container.class); + // Construct the Container-id + ContainerId cId = createContainerId(); + when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getUser()).thenReturn(user); + when(mockContainer.getResource()).thenReturn(resource); + startRequest.setContainer(mockContainer); containerManager.startContainer(startRequest); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index 29d6a4c..c263a0d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -192,7 +192,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) Resource resource = BuilderUtils.newResource(2, 1); when(mockContainer.getResource()).thenReturn(resource); Container container = - new ContainerImpl(conf, mockDispatcher, launchContext, + new ContainerImpl(conf, mockDispatcher, launchContext, "user", mockContainer, null, mockMetrics); this.context.getContainers().put(firstContainerID, container); } else if (heartBeatID == 2) { @@ -218,8 +218,8 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) Resource resource = BuilderUtils.newResource(3, 1); when(mockContainer.getResource()).thenReturn(resource); Container container = - new ContainerImpl(conf, mockDispatcher, launchContext, - mockContainer, null, mockMetrics); + new ContainerImpl(conf, mockDispatcher, launchContext, "user", + mockContainer, null, mockMetrics); this.context.getContainers().put(secondContainerID, container); } else if (heartBeatID == 3) { // Checks on the RM end diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index 8300d8f..a836f5e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -28,7 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ContainerManager; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -73,7 +73,7 @@ protected final NodeManagerMetrics metrics = NodeManagerMetrics.create(); - public BaseContainerManagerTest() throws UnsupportedFileSystemException { + public BaseContainerManagerTest() throws IOException { localFS = FileContext.getLocalFSFileContext(); localDir = new File("target", this.getClass().getName() + "-localDir") @@ -85,6 +85,7 @@ public BaseContainerManagerTest() throws UnsupportedFileSystemException { new File("target", this.getClass().getName() + "-remoteLogDir") .getAbsoluteFile(); tmpDir = new File("target", this.getClass().getName() + "-tmpDir"); + user = UserGroupInformation.getCurrentUser().getShortUserName(); } protected static Log LOG = LogFactory @@ -95,7 +96,7 @@ public BaseContainerManagerTest() throws UnsupportedFileSystemException { conf)); protected ContainerExecutor exec; protected DeletionService delSrvc; - protected String user = "nobody"; + protected final String user; protected NodeHealthCheckerService nodeHealthChecker; protected LocalDirsHandlerService dirsHandler; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index d405a7c..79763be 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -34,7 +37,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; @@ -61,11 +63,10 @@ import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.Test; -import static org.mockito.Mockito.*; public class TestContainerManager extends BaseContainerManagerTest { - public TestContainerManager() throws UnsupportedFileSystemException { + public TestContainerManager() throws IOException { super(); } @@ -119,13 +120,9 @@ public void testContainerSetup() throws IOException, InterruptedException { fileWriter.write("Hello World!"); fileWriter.close(); - ContainerLaunchContext container = recordFactory.newRecordInstance(ContainerLaunchContext.class); - // ////// Construct the Container-id ContainerId cId = createContainerId(); - container.setUser(user); - // ////// Construct the container-spec. ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); @@ -143,9 +140,9 @@ public void testContainerSetup() throws IOException, InterruptedException { new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(container.getUser()); Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getUser()).thenReturn(user); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(512, 1)); @@ -212,8 +209,6 @@ public void testContainerLaunchAndStop() throws IOException, // ////// Construct the Container-id ContainerId cId = createContainerId(); - containerLaunchContext.setUser(user); - URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(scriptFile.getAbsolutePath()))); @@ -229,13 +224,13 @@ public void testContainerLaunchAndStop() throws IOException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = new ArrayList(); commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getUser()).thenReturn(user); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(100 * 1024 * 1024, 1)); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); @@ -315,8 +310,6 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException, Interr // ////// Construct the Container-id ContainerId cId = createContainerId(); - containerLaunchContext.setUser(user); - URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(scriptFile.getAbsolutePath()))); @@ -332,13 +325,13 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException, Interr new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = new ArrayList(); commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getUser()).thenReturn(user); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(100 * 1024 * 1024, 1)); @@ -402,14 +395,10 @@ public void testLocalFilesCleanup() throws InterruptedException, fileWriter.write("Hello World!"); fileWriter.close(); - ContainerLaunchContext container = recordFactory.newRecordInstance(ContainerLaunchContext.class); - // ////// Construct the Container-id ContainerId cId = createContainerId(); ApplicationId appId = cId.getApplicationAttemptId().getApplicationId(); - container.setUser(user); - // ////// Construct the container-spec. ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); // containerLaunchContext.resources = @@ -428,9 +417,9 @@ public void testLocalFilesCleanup() throws InterruptedException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(container.getUser()); Container mockContainer = mock(Container.class); when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getUser()).thenReturn(user); when(mockContainer.getResource()).thenReturn( BuilderUtils.newResource(100, 1)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index 230ce46..d0fbe84 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -25,11 +25,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.io.IOException; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.AbstractMap.SimpleEntry; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -56,10 +54,10 @@ import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest; @@ -527,7 +525,7 @@ public boolean matches(Object o) { private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx, org.apache.hadoop.yarn.api.records.Container container) { - return new ContainerImpl(conf, disp, ctx, container, null, metrics); + return new ContainerImpl(conf, disp, ctx, "user", container, null, metrics); } @SuppressWarnings("unchecked") @@ -574,7 +572,7 @@ private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx, org.apache.hadoop.yarn.api.records.Container mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class); cId = BuilderUtils.newContainerId(appId, 1, timestamp, id); - when(ctxt.getUser()).thenReturn(this.user); + when(mockContainer.getUser()).thenReturn(this.user); when(mockContainer.getId()).thenReturn(cId); Resource resource = BuilderUtils.newResource(1024, 1); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 7027072..fe48019 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.BufferedReader; import java.io.File; @@ -34,8 +36,9 @@ import java.util.List; import java.util.Map; +import junit.framework.Assert; + import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; @@ -51,26 +54,21 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import org.junit.Before; import org.junit.Test; -import static org.mockito.Mockito.*; - -import junit.framework.Assert; public class TestContainerLaunch extends BaseContainerManagerTest { - public TestContainerLaunch() throws UnsupportedFileSystemException { + public TestContainerLaunch() throws IOException { super(); } @@ -201,8 +199,7 @@ public void testContainerEnvVariables() throws Exception { recordFactory.newRecordInstance(ContainerId.class); cId.setApplicationAttemptId(appAttemptId); when(mockContainer.getId()).thenReturn(cId); - - containerLaunchContext.setUser(user); + when(mockContainer.getUser()).thenReturn(user); // upload the script file so that the container can run it URL resource_alpha = @@ -222,7 +219,6 @@ public void testContainerEnvVariables() throws Exception { containerLaunchContext.setLocalResources(localResources); // set up the rest of the container - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = new ArrayList(); commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); @@ -319,8 +315,7 @@ public void testDelayedKill() throws Exception { recordFactory.newRecordInstance(ContainerId.class); cId.setApplicationAttemptId(appAttemptId); when(mockContainer.getId()).thenReturn(cId); - - containerLaunchContext.setUser(user); + when(mockContainer.getUser()).thenReturn(user); // upload the script file so that the container can run it URL resource_alpha = @@ -340,7 +335,6 @@ public void testDelayedKill() throws Exception { containerLaunchContext.setLocalResources(localResources); // set up the rest of the container - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = new ArrayList(); commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index ccbf9f7..5244de9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -18,9 +18,19 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation; -import static org.mockito.Mockito.*; import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyMap; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -46,16 +56,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.UnsupportedFileSystemException; -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -71,9 +78,9 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat; -import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader; +import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; @@ -107,7 +114,7 @@ private File remoteRootLogDir = new File("target", this.getClass() .getName() + "-remoteLogDir"); - public TestLogAggregationService() throws UnsupportedFileSystemException { + public TestLogAggregationService() throws IOException { super(); this.remoteRootLogDir.mkdir(); } @@ -691,8 +698,7 @@ public void testLogAggregationForRealContainerLaunch() throws IOException, ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0); when(mockContainer.getId()).thenReturn(cId); - - containerLaunchContext.setUser(this.user); + when(mockContainer.getUser()).thenReturn(this.user); URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS @@ -709,7 +715,6 @@ public void testLogAggregationForRealContainerLaunch() throws IOException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = new ArrayList(); commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index a27b357..5c9e612 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -21,7 +21,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.BufferedReader; import java.io.File; @@ -39,7 +40,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -71,7 +71,7 @@ public class TestContainersMonitor extends BaseContainerManagerTest { - public TestContainersMonitor() throws UnsupportedFileSystemException { + public TestContainersMonitor() throws IOException { super(); } @@ -198,7 +198,6 @@ public void testContainerKillOnMemoryOverflow() throws IOException, ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - Container mockContainer = mock(Container.class); // ////// Construct the Container-id ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class); @@ -211,9 +210,6 @@ public void testContainerKillOnMemoryOverflow() throws IOException, ContainerId cId = recordFactory.newRecordInstance(ContainerId.class); cId.setId(0); cId.setApplicationAttemptId(appAttemptId); - when(mockContainer.getId()).thenReturn(cId); - - containerLaunchContext.setUser(user); URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS @@ -230,16 +226,18 @@ public void testContainerKillOnMemoryOverflow() throws IOException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - containerLaunchContext.setUser(containerLaunchContext.getUser()); List commands = new ArrayList(); commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); - when(mockContainer.getResource()).thenReturn( - BuilderUtils.newResource(8 * 1024 * 1024, 1)); StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); + Container mockContainer = mock(Container.class); + when(mockContainer.getId()).thenReturn(cId); + when(mockContainer.getUser()).thenReturn(user); + when(mockContainer.getResource()).thenReturn( + BuilderUtils.newResource(8 * 1024 * 1024, 1)); startRequest.setContainer(mockContainer); containerManager.startContainer(startRequest); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index dbb50bb..cf6d7b3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -59,7 +59,6 @@ public MockContainer(ApplicationAttemptId appAttemptId, uniqId); this.launchContext = recordFactory .newRecordInstance(ContainerLaunchContext.class); - launchContext.setUser(user); this.state = ContainerState.NEW; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java index 48abd9e..af83710 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java @@ -182,10 +182,9 @@ public boolean isPmemCheckEnabled() { org.apache.hadoop.yarn.api.records.Container mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class); when(mockContainer.getId()).thenReturn(containerId); - launchContext.setUser(user); Container container = - new ContainerImpl(conf, dispatcher, launchContext, mockContainer, - null, metrics) { + new ContainerImpl(conf, dispatcher, launchContext, user, + mockContainer, null, metrics) { @Override public ContainerState getContainerState() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index bcc1f64..6fb343b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -38,7 +38,6 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; @@ -83,7 +82,6 @@ import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; @@ -266,7 +264,7 @@ public SubmitApplicationResponse submitApplication( ApplicationSubmissionContext submissionContext = request .getApplicationSubmissionContext(); ApplicationId applicationId = submissionContext.getApplicationId(); - String user = submissionContext.getAMContainerSpec().getUser(); + String user = null; try { user = UserGroupInformation.getCurrentUser().getShortUserName(); if (rmContext.getRMApps().get(applicationId) != null) { @@ -274,9 +272,6 @@ public SubmitApplicationResponse submitApplication( + " is already present! Cannot add a duplicate!"); } - // Safety - submissionContext.getAMContainerSpec().setUser(user); - // Check whether AM resource requirements are within required limits if (!submissionContext.getUnmanagedAM()) { ResourceRequest amReq = BuilderUtils.newResourceRequest( @@ -295,8 +290,8 @@ public SubmitApplicationResponse submitApplication( // This needs to be synchronous as the client can query // immediately following the submission to get the application status. // So call handle directly and do not send an event. - rmAppManager.handle(new RMAppManagerSubmitEvent(submissionContext, System - .currentTimeMillis())); + rmAppManager.handle(new RMAppManagerSubmitEvent(submissionContext, user, + System.currentTimeMillis())); LOG.info("Application with id " + applicationId.getId() + " submitted by user " + user); @@ -304,6 +299,7 @@ public SubmitApplicationResponse submitApplication( "ClientRMService", applicationId); } catch (IOException ie) { LOG.info("Exception in submitting application", ie); + user = user == null ? "UNIDENTIFIED_USER" : user; RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, ie.getMessage(), "ClientRMService", "Exception in submitting application", applicationId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 8a92ab1..ff198f0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -232,9 +232,9 @@ protected synchronized void checkAppNumCompletedLimit() { @SuppressWarnings("unchecked") protected void submitApplication( - ApplicationSubmissionContext submissionContext, long submitTime, - boolean isRecovered) { - ApplicationId applicationId = submissionContext.getApplicationId(); + ApplicationSubmissionContext submissionContext, String user, + long submitTime, boolean isRecovered) { + ApplicationId applicationId = submissionContext.getApplicationId(); RMApp application = null; try { @@ -250,11 +250,9 @@ protected void submitApplication( // Create RMApp application = new RMAppImpl(applicationId, rmContext, this.conf, - submissionContext.getApplicationName(), - submissionContext.getAMContainerSpec().getUser(), - submissionContext.getQueue(), - submissionContext, this.scheduler, this.masterService, - submitTime); + submissionContext.getApplicationName(), user, + submissionContext.getQueue(), submissionContext, this.scheduler, + this.masterService, submitTime); // Sanity check - duplicate? if (rmContext.getRMApps().putIfAbsent(applicationId, application) != @@ -348,8 +346,8 @@ public void recover(RMState state) throws Exception { if(shouldRecover) { LOG.info("Recovering application " + appState.getAppId()); - submitApplication(appState.getApplicationSubmissionContext(), - appState.getSubmitTime(), true); + submitApplication(appState.getApplicationSubmissionContext(), + appState.getUser(), appState.getSubmitTime(), true); // re-populate attempt information in application RMAppImpl appImpl = (RMAppImpl) rmContext.getRMApps().get( appState.getAppId()); @@ -377,10 +375,12 @@ public void handle(RMAppManagerEvent event) { break; case APP_SUBMIT: { + RMAppManagerSubmitEvent submitEvent = (RMAppManagerSubmitEvent)event; ApplicationSubmissionContext submissionContext = - ((RMAppManagerSubmitEvent)event).getSubmissionContext(); - long submitTime = ((RMAppManagerSubmitEvent)event).getSubmitTime(); - submitApplication(submissionContext, submitTime, false); + submitEvent.getSubmissionContext(); + String user = submitEvent.getUser(); + long submitTime = submitEvent.getSubmitTime(); + submitApplication(submissionContext, user, submitTime, false); } break; default: diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManagerSubmitEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManagerSubmitEvent.java index afcd24d..cecda75 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManagerSubmitEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManagerSubmitEvent.java @@ -23,20 +23,27 @@ public class RMAppManagerSubmitEvent extends RMAppManagerEvent { private final ApplicationSubmissionContext submissionContext; + private final String user; private final long submitTime; public RMAppManagerSubmitEvent( - ApplicationSubmissionContext submissionContext, long submitTime) { + ApplicationSubmissionContext submissionContext, String user, + long submitTime) { super(submissionContext.getApplicationId(), - RMAppManagerEventType.APP_SUBMIT); + RMAppManagerEventType.APP_SUBMIT); this.submissionContext = submissionContext; + this.user = user; this.submitTime = submitTime; } public ApplicationSubmissionContext getSubmissionContext() { return this.submissionContext; } - + + public String getUser() { + return this.user; + } + public long getSubmitTime() { return this.submitTime; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 517d7fb..dde1f1f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -166,7 +166,6 @@ private ContainerLaunchContext createAMContainerLaunchContext( new String[0]))); // Finalize the container - container.setUser(applicationMasterContext.getAMContainerSpec().getUser()); setupTokensAndEnv(container, containerID); return container; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index aca84ad..da38267 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -99,9 +99,10 @@ public synchronized RMState loadState() throws Exception { ApplicationStateDataPBImpl appStateData = new ApplicationStateDataPBImpl( ApplicationStateDataProto.parseFrom(childData)); - ApplicationState appState = new ApplicationState( - appStateData.getSubmitTime(), - appStateData.getApplicationSubmissionContext()); + ApplicationState appState = + new ApplicationState(appStateData.getSubmitTime(), + appStateData.getUser(), + appStateData.getApplicationSubmissionContext()); // assert child node name is same as actual applicationId assert appId.equals(appState.context.getApplicationId()); state.appState.put(appId, appState); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index 4b398d4..8c59f28 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -62,9 +62,10 @@ protected synchronized void closeInternal() throws Exception { public void storeApplicationState(String appId, ApplicationStateDataPBImpl appStateData) throws Exception { - ApplicationState appState = new ApplicationState( - appStateData.getSubmitTime(), - appStateData.getApplicationSubmissionContext()); + ApplicationState appState = + new ApplicationState(appStateData.getSubmitTime(), + appStateData.getUser(), + appStateData.getApplicationSubmissionContext()); if (state.appState.containsKey(appState.getAppId())) { Exception e = new IOException("App: " + appId + " is already stored."); LOG.info("Error storing info for app: " + appId, e); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index 87a2608..502f0bb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppStoredEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStoredEvent; +import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; @Private @Unstable @@ -78,12 +79,15 @@ public ApplicationAttemptId getAttemptId() { */ public static class ApplicationState { final ApplicationSubmissionContext context; + final String user; final long submitTime; Map attempts = new HashMap(); - ApplicationState(long submitTime, ApplicationSubmissionContext context) { + ApplicationState(long submitTime, String user, + ApplicationSubmissionContext context) { this.submitTime = submitTime; + this.user = user; this.context = context; } @@ -93,6 +97,9 @@ public ApplicationId getAppId() { public long getSubmitTime() { return submitTime; } + public String getUser() { + return this.user; + } public int getAttemptCount() { return attempts.size(); } @@ -177,8 +184,8 @@ public synchronized void storeApplication(RMApp app) { ApplicationSubmissionContext context = app .getApplicationSubmissionContext(); assert context instanceof ApplicationSubmissionContextPBImpl; - ApplicationState appState = new ApplicationState( - app.getSubmitTime(), context); + ApplicationState appState = + new ApplicationState(app.getSubmitTime(), app.getUser(), context); dispatcher.getEventHandler().handle(new RMStateStoreAppEvent(appState)); } @@ -223,8 +230,9 @@ protected abstract void storeApplicationAttemptState(String attemptId, * There is no notification of completion for this operation. */ public synchronized void removeApplication(RMApp app) { - ApplicationState appState = new ApplicationState( - app.getSubmitTime(), app.getApplicationSubmissionContext()); + ApplicationState appState = + new ApplicationState(app.getSubmitTime(), app.getUser(), + app.getApplicationSubmissionContext()); for(RMAppAttempt appAttempt : app.getAppAttempts().values()) { ApplicationAttemptState attemptState = new ApplicationAttemptState( appAttempt.getAppAttemptId(), appAttempt.getMasterContainer()); @@ -256,16 +264,15 @@ private synchronized void handleStoreEvent(RMStateStoreEvent event) { switch(event.getType()) { case STORE_APP: { - ApplicationState apptState = + ApplicationState appState = ((RMStateStoreAppEvent) event).getAppState(); Exception storedException = null; ApplicationStateDataPBImpl appStateData = - new ApplicationStateDataPBImpl(); - appStateData.setSubmitTime(apptState.getSubmitTime()); - appStateData.setApplicationSubmissionContext( - apptState.getApplicationSubmissionContext()); + YarnServerBuilderUtils.newApplicationStateData( + appState.getSubmitTime(), appState.getUser(), + appState.getApplicationSubmissionContext()); ApplicationId appId = - apptState.getApplicationSubmissionContext().getApplicationId(); + appState.getApplicationSubmissionContext().getApplicationId(); LOG.info("Storing info for app: " + appId); try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 7b63cbe..fcd11df 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -574,7 +574,7 @@ private void createNewAttempt(boolean startAttempt) { RMAppAttempt attempt = new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService, - submissionContext, conf); + submissionContext, user, conf); attempts.put(appAttemptId, attempt); currentAttempt = attempt; if(startAttempt) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index eaa15f5..54706da 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -125,12 +125,15 @@ private final ApplicationAttemptId applicationAttemptId; private ClientToken clientToken; private final ApplicationSubmissionContext submissionContext; + private final String user; //nodes on while this attempt's containers ran private final Set ranNodes = new HashSet(); private final List justFinishedContainers = new ArrayList(); + // The container corresponding to the AM. Non-null only after the AM container + // is allocated by the scheduler private Container masterContainer; private float progress = 0; @@ -356,13 +359,14 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, RMContext rmContext, YarnScheduler scheduler, ApplicationMasterService masterService, - ApplicationSubmissionContext submissionContext, + ApplicationSubmissionContext submissionContext, String user, Configuration conf) { this.conf = conf; this.applicationAttemptId = appAttemptId; this.rmContext = rmContext; this.eventHandler = rmContext.getDispatcher().getEventHandler(); this.submissionContext = submissionContext; + this.user = user; this.scheduler = scheduler; this.masterService = masterService; @@ -687,10 +691,9 @@ public void transition(RMAppAttemptImpl appAttempt, .registerAppAttempt(appAttempt.applicationAttemptId); // Add the application to the scheduler - appAttempt.eventHandler.handle( - new AppAddedSchedulerEvent(appAttempt.applicationAttemptId, - appAttempt.submissionContext.getQueue(), - appAttempt.submissionContext.getAMContainerSpec().getUser())); + appAttempt.eventHandler.handle(new AppAddedSchedulerEvent( + appAttempt.applicationAttemptId, appAttempt.submissionContext + .getQueue(), appAttempt.user)); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 58dcb73..c8ff18e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -633,7 +633,8 @@ public void submitApplication(FiCaSchedulerApp application, String userName, // Careful! Locking order is important! // Check queue ACLs - UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(userName); + UserGroupInformation userUgi = + UserGroupInformation.createRemoteUser(userName); if (!hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi)) { throw new AccessControlException("User " + userName + " cannot submit" + " applications to queue " + getQueuePath()); @@ -1241,9 +1242,9 @@ Container createContainer(FiCaSchedulerApp application, FiCaSchedulerNode node, .getApplicationAttemptId(), application.getNewContainerId()); // Create the container - Container container = BuilderUtils.newContainer(containerId, nodeId, - node.getRMNode().getHttpAddress(), capability, priority, - null); + Container container = + BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() + .getHttpAddress(), capability, priority, application.getUser(), null); return container; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java index fa01a0b..c69ed14 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java @@ -167,9 +167,10 @@ public Container createContainer( } // Create the container - Container container = BuilderUtils.newContainer(containerId, nodeId, - node.getRMNode().getHttpAddress(), capability, priority, - containerToken); + Container container = + BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() + .getHttpAddress(), capability, priority, application.getUser(), + containerToken); return container; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index d5a5427..5123bda 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -563,9 +563,10 @@ private int assignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application } // Create the container - Container container = BuilderUtils.newContainer(containerId, nodeId, - node.getRMNode().getHttpAddress(), capability, priority, - containerToken); + Container container = + BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() + .getHttpAddress(), capability, priority, application.getUser(), + containerToken); // Allocate! diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java index 875212f..d4db598 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java @@ -56,11 +56,11 @@ public AppAttemptInfo(RMAppAttempt attempt) { this.containerId = masterContainer.getId().toString(); this.nodeHttpAddress = masterContainer.getNodeHttpAddress(); this.nodeId = masterContainer.getNodeId().toString(); - this.logsLink = join(HttpConfig.getSchemePrefix(), - masterContainer.getNodeHttpAddress(), - "/node", "/containerlogs/", - ConverterUtils.toString(masterContainer.getId()), "/", - attempt.getSubmissionContext().getAMContainerSpec().getUser()); + this.logsLink = + join(HttpConfig.getSchemePrefix(), + masterContainer.getNodeHttpAddress(), "/node", "/containerlogs/", + ConverterUtils.toString(masterContainer.getId()), "/", + masterContainer.getUser()); } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index 3ce45ac..b0ee24c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -34,6 +35,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; @@ -127,15 +129,21 @@ public Resource getUsedResources() { return used; } - public synchronized void submit() throws IOException { + public synchronized void submit() throws IOException, InterruptedException { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(this.applicationId); - context.getAMContainerSpec().setUser(this.user); context.setQueue(this.queue); - SubmitApplicationRequest request = recordFactory + final SubmitApplicationRequest request = recordFactory .newRecordInstance(SubmitApplicationRequest.class); request.setApplicationSubmissionContext(context); - resourceManager.getClientRMService().submitApplication(request); + UserGroupInformation.createRemoteUser(this.user).doAs( + new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException { + resourceManager.getClientRMService().submitApplication(request); + return null; + } + }); } public synchronized void addResourceRequestSpec( @@ -340,7 +348,8 @@ private synchronized void assign(Priority priority, NodeType type, // Launch the container StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(createCLC()); + startRequest.setContainerLaunchContext(recordFactory + .newRecordInstance(ContainerLaunchContext.class)); startRequest.setContainer(container); nodeManager.startContainer(startRequest); break; @@ -396,10 +405,4 @@ private void updateResourceRequest(ResourceRequest request) { + " request=" + request); } } - - private ContainerLaunchContext createCLC() { - ContainerLaunchContext clc = recordFactory.newRecordInstance(ContainerLaunchContext.class); - clc.setUser(this.user); - return clc; - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index e39f303..a2edd0b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -174,7 +174,6 @@ public RMApp submitApp(int masterMemory, String name, String user, capability.setMemory(masterMemory); sub.setResource(capability); clc.setApplicationACLs(acls); - clc.setUser(user); sub.setAMContainerSpec(clc); req.setApplicationSubmissionContext(sub); UserGroupInformation fakeUser = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index dba5acd..e1fef33 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -187,7 +187,7 @@ synchronized public StartContainerResponse startContainer( BuilderUtils.newContainer(requestContainer.getId(), this.nodeId, nodeHttpAddress, requestContainer.getResource(), - null, null // DKDC - Doesn't matter + null, null, null // DKDC - Doesn't matter ); ContainerStatus containerStatus = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index f5cc7d3..d8700c7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -162,10 +162,11 @@ public int getCompletedAppsListSize() { public void setCompletedAppsMax(int max) { super.setCompletedAppsMax(max); } - public void submitApplication( - ApplicationSubmissionContext submissionContext) { - super.submitApplication( - submissionContext, System.currentTimeMillis(), false); + + public void + submitApplication(ApplicationSubmissionContext submissionContext) { + super.submitApplication(submissionContext, "user", + System.currentTimeMillis(), false); } } @@ -507,7 +508,7 @@ public void testRMAppSubmitError() throws Exception { ContainerLaunchContext clc = BuilderUtils.newContainerLaunchContext(null, null, null, null, null, - null, null); + null); context.setAMContainerSpec(clc); // our testApp1 should be rejected and original app with same id should be left in place appMonitor.submitApplication(context); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index aa7af9c..3a93b56 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -360,7 +360,6 @@ public void testInvalidResourceRequestWhenSubmittingApplication() } private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId) { - String user = MockApps.newUserName(); String queue = MockApps.newQueue(); ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); @@ -370,7 +369,6 @@ private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId) { ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class); when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec); - when(submissionContext.getAMContainerSpec().getUser()).thenReturn(user); when(submissionContext.getQueue()).thenReturn(queue); when(submissionContext.getApplicationId()).thenReturn(appId); when(submissionContext.getResource()).thenReturn(resource); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index af95d2b..9b2dd1c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -67,7 +67,7 @@ public void tearDown() throws Exception { } // @Test - public void testResourceAllocation() throws IOException { + public void testResourceAllocation() throws IOException, InterruptedException { LOG.info("--- START: testResourceAllocation ---"); final int memory = 4 * 1024; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java index 440908f..ce1d5ef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -181,6 +182,8 @@ void storeApp(RMStateStore store, ApplicationId appId, long time) RMApp mockApp = mock(RMApp.class); when(mockApp.getApplicationId()).thenReturn(appId); when(mockApp.getSubmitTime()).thenReturn(time); + when(mockApp.getUser()).thenReturn( + UserGroupInformation.getCurrentUser().getShortUserName()); when(mockApp.getApplicationSubmissionContext()).thenReturn(context); store.storeApplication(mockApp); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index f736edf2..093bb34 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -203,8 +203,8 @@ public void setUp() throws Exception { when(submissionContext.getQueue()).thenReturn(queue); Resource resource = BuilderUtils.newResource(1536, 1); ContainerLaunchContext amContainerSpec = - BuilderUtils.newContainerLaunchContext(user, null, null, - null, null, null, null); + BuilderUtils.newContainerLaunchContext(null, null, null, null, null, + null); when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec); when(submissionContext.getResource()).thenReturn(resource); @@ -213,7 +213,7 @@ public void setUp() throws Exception { application = mock(RMApp.class); applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, rmContext, scheduler, - masterService, submissionContext, new Configuration()); + masterService, submissionContext, user, new Configuration()); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index 6e2b834..fe4d1d2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -69,7 +69,7 @@ public void testReleaseWhileRunning() { Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, - "host:3465", resource, priority, null); + "host:3465", resource, priority, "user", null); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, eventHandler, expirer); @@ -139,7 +139,7 @@ public void testExpireWhileRunning() { Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, - "host:3465", resource, priority, null); + "host:3465", resource, priority, "user", null); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, eventHandler, expirer); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 039e0b0..060fef6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -1456,7 +1456,6 @@ public void testAssignContainer() throws Exception { } } - @SuppressWarnings("unchecked") @Test public void testNotAllowSubmitApplication() throws Exception { // Set acl's @@ -1483,8 +1482,8 @@ public void testNotAllowSubmitApplication() throws Exception { new ApplicationMasterService(resourceManager.getRMContext(), scheduler); ApplicationSubmissionContext submissionContext = new ApplicationSubmissionContextPBImpl(); ContainerLaunchContext clc = - BuilderUtils.newContainerLaunchContext(user, null, null, null, null, - null, null); + BuilderUtils.newContainerLaunchContext(null, null, null, null, null, + null); submissionContext.setApplicationId(applicationId); submissionContext.setAMContainerSpec(clc); RMApp application = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 1d405be..03c75e3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1081,7 +1081,7 @@ public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id, logsLink.startsWith("http://")); assertTrue( "logsLink doesn't contain user info", logsLink.endsWith("/" - + appAttempt.getSubmissionContext().getAMContainerSpec().getUser())); + + appAttempt.getMasterContainer().getUser())); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index 69e197a..2fd21a2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -146,6 +146,7 @@ public void testAuthenticatedUser() throws IOException, // Now request a container. final Container allocatedContainer = requestAndGetContainer(scheduler, appID); + Assert.assertEquals("testUser", allocatedContainer.getUser()); // Now talk to the NM for launching the container. final ContainerId containerID = allocatedContainer.getId(); @@ -289,7 +290,7 @@ public void testUnauthorizedUser() throws IOException, InterruptedException { final ContainerId containerID = allocatedContainer.getId(); /////////// Test calls with illegal containerIDs and illegal Resources - UserGroupInformation unauthorizedUser = UserGroupInformation + UserGroupInformation maliciousUser = UserGroupInformation .createRemoteUser(containerID.toString()); ContainerToken containerToken = allocatedContainer.getContainerToken(); @@ -303,9 +304,9 @@ public void testUnauthorizedUser() throws IOException, InterruptedException { identifierBytes, containerToken.getPassword().array(), new Text( containerToken.getKind()), new Text(containerToken.getService())); - unauthorizedUser.addToken(token); + maliciousUser.addToken(token); ContainerManager client = - unauthorizedUser.doAs(new PrivilegedAction() { + maliciousUser.doAs(new PrivilegedAction() { @Override public ContainerManager run() { ContainerManager client = (ContainerManager) yarnRPC.getProxy( @@ -316,7 +317,7 @@ public ContainerManager run() { LOG.info("Going to contact NM: unauthorized request"); callWithIllegalContainerID(client, tokenId); - callWithIllegalResource(client, tokenId); + callWithIllegalResource(client, allocatedContainer, tokenId); callWithIllegalUserName(client, tokenId); return client; @@ -328,7 +329,7 @@ public ContainerManager run() { /////////// Test calls with expired tokens RPC.stopProxy(client); - unauthorizedUser = UserGroupInformation + maliciousUser = UserGroupInformation .createRemoteUser(containerID.toString()); RMContainerTokenSecretManager containerTokenSecreteManager = @@ -346,8 +347,8 @@ public ContainerManager run() { newTokenId.getBytes(), passowrd, new Text( containerToken.getKind()), new Text(containerToken.getService())); - unauthorizedUser.addToken(token); - unauthorizedUser.doAs(new PrivilegedAction() { + maliciousUser.addToken(token); + maliciousUser.doAs(new PrivilegedAction() { @Override public Void run() { ContainerManager client = (ContainerManager) yarnRPC.getProxy( @@ -359,8 +360,9 @@ public Void run() { ContainerLaunchContext context = createContainerLaunchContextForTest(newTokenId); Container container = BuilderUtils.newContainer(newTokenId.getContainerID(), null, null, - BuilderUtils.newResource(newTokenId.getResource().getMemory(), - newTokenId.getResource().getVirtualCores()), null, null); + BuilderUtils.newResource(newTokenId.getResource().getMemory(), + newTokenId.getResource().getVirtualCores()), null, newTokenId + .getApplicationSubmitter(), null); StartContainerRequest request = Records.newRecord(StartContainerRequest.class); request.setContainerLaunchContext(context); request.setContainer(container); @@ -397,7 +399,7 @@ public Void run() { } private AMRMProtocol submitAndRegisterApplication( - ResourceManager resourceManager, final YarnRPC yarnRPC, + final ResourceManager resourceManager, final YarnRPC yarnRPC, ApplicationId appID) throws IOException, UnsupportedFileSystemException, YarnRemoteException, InterruptedException { @@ -407,24 +409,30 @@ private AMRMProtocol submitAndRegisterApplication( Arrays.asList("ping", "-n", "100", "127.0.0.1", ">nul") : Arrays.asList("sleep", "100"); - ContainerLaunchContext amContainer = - BuilderUtils.newContainerLaunchContext("testUser", - Collections. emptyMap(), - new HashMap(), cmd, - new HashMap(), null, - new HashMap()); + ContainerLaunchContext amContainerLaunchContext = + BuilderUtils.newContainerLaunchContext( + Collections. emptyMap(), + new HashMap(), cmd, + new HashMap(), null, + new HashMap()); ApplicationSubmissionContext appSubmissionContext = recordFactory .newRecordInstance(ApplicationSubmissionContext.class); appSubmissionContext.setApplicationId(appID); - appSubmissionContext.setAMContainerSpec(amContainer); - appSubmissionContext.getAMContainerSpec().setUser("testUser"); + appSubmissionContext.setAMContainerSpec(amContainerLaunchContext); appSubmissionContext.setResource(BuilderUtils.newResource(1024, 1)); - SubmitApplicationRequest submitRequest = recordFactory + final SubmitApplicationRequest submitRequest = recordFactory .newRecordInstance(SubmitApplicationRequest.class); submitRequest.setApplicationSubmissionContext(appSubmissionContext); - resourceManager.getClientRMService().submitApplication(submitRequest); + UserGroupInformation.createRemoteUser("testUser").doAs( + new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + resourceManager.getClientRMService().submitApplication(submitRequest); + return null; + } + }); // Wait till container gets allocated for AM int waitCounter = 0; @@ -446,6 +454,9 @@ private AMRMProtocol submitAndRegisterApplication( Assert.assertNotNull(state); Assert.assertEquals(RMAppAttemptState.LAUNCHED, state); + Container amContainer = appAttempt.getMasterContainer(); + Assert.assertEquals("testUser", amContainer.getUser()); + UserGroupInformation currentUser = UserGroupInformation.createRemoteUser( appAttempt.getAppAttemptId().toString()); @@ -539,7 +550,7 @@ void callWithIllegalContainerID(ContainerManager client, } void callWithIllegalResource(ContainerManager client, - ContainerTokenIdentifier tokenId) { + Container originalAllocatedContainer, ContainerTokenIdentifier tokenId) { StartContainerRequest request = recordFactory .newRecordInstance(StartContainerRequest.class); // Authenticated but unauthorized, due to wrong resource @@ -547,7 +558,8 @@ void callWithIllegalResource(ContainerManager client, createContainerLaunchContextForTest(tokenId); Container container = BuilderUtils.newContainer(tokenId.getContainerID(), null, null, - BuilderUtils.newResource(2048, 1), null, null); + BuilderUtils.newResource(2048, 1), null, + originalAllocatedContainer.getUser(), null); request.setContainerLaunchContext(context); request.setContainer(container); try { @@ -571,11 +583,11 @@ void callWithIllegalUserName(ContainerManager client, // Authenticated but unauthorized, due to wrong resource ContainerLaunchContext context = createContainerLaunchContextForTest(tokenId); - context.setUser("Saruman"); // Set a different user-name. + // Create a container with a different user-name. Container container = BuilderUtils.newContainer(tokenId.getContainerID(), null, null, - BuilderUtils.newResource(tokenId.getResource().getMemory(), tokenId - .getResource().getVirtualCores()), null, null); + BuilderUtils.newResource(tokenId.getResource().getMemory(), tokenId + .getResource().getVirtualCores()), null, "Saruman", null); request.setContainerLaunchContext(context); request.setContainer(container); try { @@ -588,7 +600,7 @@ void callWithIllegalUserName(ContainerManager client, "Unauthorized request to start container. ")); Assert.assertTrue(e.getMessage().contains( "Expected user-name " + tokenId.getApplicationSubmitter() - + " but found " + context.getUser())); + + " but found Saruman")); } } @@ -596,10 +608,9 @@ private ContainerLaunchContext createContainerLaunchContextForTest( ContainerTokenIdentifier tokenId) { ContainerLaunchContext context = BuilderUtils.newContainerLaunchContext( - "testUser", new HashMap(), - new HashMap(), new ArrayList(), - new HashMap(), null, - new HashMap()); + new HashMap(), new HashMap(), + new ArrayList(), new HashMap(), null, + new HashMap()); return context; } }