diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java index d13159b..bfb4890 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java @@ -35,14 +35,23 @@ private final String user; private final ContainerId containerId; private final Resource resource; + private final boolean isAMContainer; @Private @Unstable public ContainerContext(String user, ContainerId containerId, Resource resource) { + this(user, containerId, resource, false); + } + + @Private + @Unstable + public ContainerContext(String user, ContainerId containerId, + Resource resource, boolean isAMContainer) { this.user = user; this.containerId = containerId; this.resource = resource; + this.isAMContainer = isAMContainer; } /** @@ -72,4 +81,13 @@ public ContainerId getContainerId() { public Resource getResource() { return resource; } + + /** + * Get whether the container is an AM or not. + * + * @return true if the container is an AM, false otherwise. + */ + public boolean isAMContainer() { + return isAMContainer; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java index 5b5bbda..d1accbc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java @@ -41,4 +41,11 @@ public ContainerInitializationContext(String user, ContainerId containerId, super(user, containerId, resource); } + @Private + @Unstable + public ContainerInitializationContext(String user, ContainerId containerId, + Resource resource, boolean isAMContainer) { + super(user, containerId, resource, isAMContainer); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java index 34ba73e..0605a44 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java @@ -41,4 +41,11 @@ public ContainerTerminationContext(String user, ContainerId containerId, super(user, containerId, resource); } + @Private + @Unstable + public ContainerTerminationContext(String user, ContainerId containerId, + Resource resource, boolean isAMContainer) { + super(user, containerId, resource, isAMContainer); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java index 9a60d01..db898db 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java @@ -66,14 +66,24 @@ public ContainerTokenIdentifier(ContainerId containerID, int masterKeyId, long rmIdentifier, Priority priority, long creationTime) { this(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, null, - CommonNodeLabelsManager.NO_LABEL); + CommonNodeLabelsManager.NO_LABEL, false); } public ContainerTokenIdentifier(ContainerId containerID, String hostName, String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, long rmIdentifier, Priority priority, long creationTime, LogAggregationContext logAggregationContext, String nodeLabelExpression) { - ContainerTokenIdentifierProto.Builder builder = + this(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, + rmIdentifier, priority, creationTime, logAggregationContext, + nodeLabelExpression, false); + } + + public ContainerTokenIdentifier(ContainerId containerID, String hostName, + String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId, + long rmIdentifier, Priority priority, long creationTime, + LogAggregationContext logAggregationContext, String nodeLabelExpression, + boolean isAMContainer) { + ContainerTokenIdentifierProto.Builder builder = ContainerTokenIdentifierProto.newBuilder(); if (containerID != null) { builder.setContainerId(((ContainerIdPBImpl)containerID).getProto()); @@ -99,7 +109,8 @@ public ContainerTokenIdentifier(ContainerId containerID, String hostName, if (nodeLabelExpression != null) { builder.setNodeLabelExpression(nodeLabelExpression); } - + builder.setIsAMContainer(isAMContainer); + proto = builder.build(); } @@ -168,6 +179,15 @@ public LogAggregationContext getLogAggregationContext() { return new LogAggregationContextPBImpl(proto.getLogAggregationContext()); } + /** + * Get whether the container is an AM or not. + * + * @return true if the container is an AM, false otherwise. + */ + public boolean getIsAMContainer() { + return proto.getIsAMContainer(); + } + @Override public void write(DataOutput out) throws IOException { LOG.debug("Writing ContainerTokenIdentifier to RPC layer: " + this); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto index d1bef21..ac8604c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto @@ -50,6 +50,7 @@ message ContainerTokenIdentifierProto { optional int64 creationTime = 9; optional LogAggregationContextProto logAggregationContext = 10; optional string nodeLabelExpression = 11; + optional bool isAMContainer = 12 [default = false]; } message ClientToAMTokenIdentifierProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java index 5fe75bc..29f2169 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto; import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; @@ -201,6 +202,11 @@ public void testContainerTokenIdentifier() throws IOException { anotherToken.getCreationTime(), creationTime); Assert.assertNull(anotherToken.getLogAggregationContext()); + + Assert.assertEquals(CommonNodeLabelsManager.NO_LABEL, + anotherToken.getNodeLabelExpression()); + + Assert.assertFalse(anotherToken.getIsAMContainer()); } @Test @@ -347,4 +353,47 @@ public void testParseTimelineDelegationTokenIdentifierRenewer() throws IOExcepti Assert.assertEquals(new Text("yarn"), token.getRenewer()); } + @Test + public void testAMContainerTokenIdentifier() throws IOException { + ContainerId containerID = ContainerId.newContainerId( + ApplicationAttemptId.newInstance(ApplicationId.newInstance( + 1, 1), 1), 1); + String hostName = "host0"; + String appSubmitter = "usr0"; + Resource r = Resource.newInstance(1024, 1); + long expiryTimeStamp = 1000; + int masterKeyId = 1; + long rmIdentifier = 1; + Priority priority = Priority.newInstance(1); + long creationTime = 1000; + + ContainerTokenIdentifier token = + new ContainerTokenIdentifier(containerID, hostName, appSubmitter, r, + expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, + null, CommonNodeLabelsManager.NO_LABEL, true); + + ContainerTokenIdentifier anotherToken = new ContainerTokenIdentifier(); + + byte[] tokenContent = token.getBytes(); + DataInputBuffer dib = new DataInputBuffer(); + dib.reset(tokenContent, tokenContent.length); + anotherToken.readFields(dib); + + Assert.assertTrue(anotherToken.getIsAMContainer()); + + token = + new ContainerTokenIdentifier(containerID, hostName, appSubmitter, r, + expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, + null, CommonNodeLabelsManager.NO_LABEL, false); + + anotherToken = new ContainerTokenIdentifier(); + + tokenContent = token.getBytes(); + dib = new DataInputBuffer(); + dib.reset(tokenContent, tokenContent.length); + anotherToken.readFields(dib); + + Assert.assertFalse(anotherToken.getIsAMContainer()); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index fb6f79b..78ed1a6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -225,7 +225,8 @@ public void handle(AuxServicesEvent event) { try { serv.initializeContainer(new ContainerInitializationContext( event.getUser(), event.getContainer().getContainerId(), - event.getContainer().getResource())); + event.getContainer().getResource(), event.getContainer() + .getContainerTokenIdentifier().getIsAMContainer())); } catch (Throwable th) { logWarningWhenAuxServiceThrowExceptions(serv, AuxServicesEventType.CONTAINER_INIT, th); @@ -237,7 +238,8 @@ public void handle(AuxServicesEvent event) { try { serv.stopContainer(new ContainerTerminationContext( event.getUser(), event.getContainer().getContainerId(), - event.getContainer().getResource())); + event.getContainer().getResource(), event.getContainer() + .getContainerTokenIdentifier().getIsAMContainer())); } catch (Throwable th) { logWarningWhenAuxServiceThrowExceptions(serv, AuxServicesEventType.CONTAINER_STOP, th); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/TestPrivilegedOperationExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/TestPrivilegedOperationExecutor.java index 7154d03..ca69a8d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/TestPrivilegedOperationExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/TestPrivilegedOperationExecutor.java @@ -217,8 +217,8 @@ public void testSquashCGroupOperationsWithValidOperations() { .squashCGroupOperations(ops); String expected = new StringBuffer (PrivilegedOperation.CGROUP_ARG_PREFIX) - .append(cGroupTasks1).append(',') - .append(cGroupTasks2).append(',') + .append(cGroupTasks1).append('%') + .append(cGroupTasks2).append('%') .append(cGroupTasks3).toString(); //We expect axactly one argument diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index dbc3cb5..8e83052 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -467,13 +467,26 @@ public ContainersAndNMTokensAllocation(List containerList, .hasNext();) { RMContainer rmContainer = i.next(); Container container = rmContainer.getContainer(); + boolean isAMContainer = false; + // The working knowledge is that masterContainer for AM is null as it + // itself is the master container. + RMAppAttempt appAttempt = + rmContext + .getRMApps() + .get( + container.getId().getApplicationAttemptId() + .getApplicationId()).getCurrentAppAttempt(); + if (appAttempt.getMasterContainer() == null + && appAttempt.getSubmissionContext().getUnmanagedAM() == false) { + isAMContainer = true; + } try { // create container token and NMToken altogether. container.setContainerToken(rmContext.getContainerTokenSecretManager() .createContainerToken(container.getId(), container.getNodeId(), getUser(), container.getResource(), container.getPriority(), rmContainer.getCreationTime(), this.logAggregationContext, - rmContainer.getNodeLabelExpression())); + rmContainer.getNodeLabelExpression(), isAMContainer)); NMToken nmToken = rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(), getApplicationAttemptId(), container); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index e5b44a6..598f279 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -829,7 +829,8 @@ public synchronized CSAssignment assignContainers(Resource clusterResource, RMAppAttempt rmAppAttempt = csContext.getRMContext().getRMApps() .get(application.getApplicationId()).getCurrentAppAttempt(); - if (null == rmAppAttempt.getMasterContainer()) { + if (rmAppAttempt.getSubmissionContext().getUnmanagedAM() == false + && null == rmAppAttempt.getMasterContainer()) { if (LOG.isDebugEnabled()) { LOG.debug("Skip allocating AM container to app_attempt=" + application.getApplicationAttemptId() diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java index 1c0533d..d9d2f41 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java @@ -166,7 +166,7 @@ public void run() { /** * Helper function for creating ContainerTokens - * + * * @param containerId * @param nodeId * @param appSubmitter @@ -179,12 +179,12 @@ public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime) { return createContainerToken(containerId, nodeId, appSubmitter, capability, - priority, createTime, null, null); + priority, createTime, null, null, false); } /** * Helper function for creating ContainerTokens - * + * * @param containerId * @param nodeId * @param appSubmitter @@ -192,12 +192,14 @@ public Token createContainerToken(ContainerId containerId, NodeId nodeId, * @param priority * @param createTime * @param logAggregationContext + * @param nodeLabelExpression + * @param isAMContainer * @return the container-token */ public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime, LogAggregationContext logAggregationContext, - String nodeLabelExpression) { + String nodeLabelExpression, boolean isAMContainer) { byte[] password; ContainerTokenIdentifier tokenIdentifier; long expiryTimeStamp = @@ -211,7 +213,7 @@ public Token createContainerToken(ContainerId containerId, NodeId nodeId, appSubmitter, capability, expiryTimeStamp, this.currentMasterKey .getMasterKey().getKeyId(), ResourceManager.getClusterTimeStamp(), priority, createTime, - logAggregationContext, nodeLabelExpression); + logAggregationContext, nodeLabelExpression, isAMContainer); password = this.createPassword(tokenIdentifier); } finally { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index 9b7eb84..e62f7d7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -55,6 +55,8 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.Task.State; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -168,6 +170,16 @@ public synchronized void submit() throws IOException, YarnException { resourceManager.getClientRMService().submitApplication(request); + RMAppEvent event = + new RMAppEvent(this.applicationId, RMAppEventType.START); + resourceManager.getRMContext().getRMApps().get(applicationId).handle(event); + event = + new RMAppEvent(this.applicationId, RMAppEventType.APP_NEW_SAVED); + resourceManager.getRMContext().getRMApps().get(applicationId).handle(event); + event = + new RMAppEvent(this.applicationId, RMAppEventType.APP_ACCEPTED); + resourceManager.getRMContext().getRMApps().get(applicationId).handle(event); + // Notify scheduler AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId, this.queue, "user"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 0ea993f..28c6832 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -286,11 +286,11 @@ protected RMContainerTokenSecretManager createContainerTokenSecretManager( public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime, - LogAggregationContext logAggregationContext, String nodeLabelExp) { + LogAggregationContext logAggregationContext, String nodeLabelExp, boolean isAMContainer) { numRetries++; return super.createContainerToken(containerId, nodeId, appSubmitter, capability, priority, createTime, logAggregationContext, - nodeLabelExp); + nodeLabelExp, isAMContainer); } }; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java index 23b708a..403c8ea 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -38,6 +39,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; @@ -152,6 +155,11 @@ protected ApplicationAttemptId createSchedulingRequest( when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt); when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn( new RMAppAttemptMetrics(id, resourceManager.getRMContext())); + ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class); + when(submissionContext.getUnmanagedAM()).thenReturn(false); + when(rmAppAttempt.getSubmissionContext()).thenReturn(submissionContext); + Container container = mock(Container.class); + when(rmAppAttempt.getMasterContainer()).thenReturn(container); resourceManager.getRMContext().getRMApps() .put(id.getApplicationId(), rmApp); @@ -175,6 +183,9 @@ protected ApplicationAttemptId createSchedulingRequest(String queueId, when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt); when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn( new RMAppAttemptMetrics(id,resourceManager.getRMContext())); + ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class); + when(submissionContext.getUnmanagedAM()).thenReturn(false); + when(rmAppAttempt.getSubmissionContext()).thenReturn(submissionContext); resourceManager.getRMContext().getRMApps() .put(id.getApplicationId(), rmApp); @@ -206,13 +217,20 @@ protected void createSchedulingRequestExistingApplication( protected void createApplicationWithAMResource(ApplicationAttemptId attId, String queue, String user, Resource amResource) { RMContext rmContext = resourceManager.getRMContext(); - RMApp rmApp = new RMAppImpl(attId.getApplicationId(), rmContext, conf, - null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, amResource, null), null, null, + ApplicationId appId = attId.getApplicationId(); + RMApp rmApp = new RMAppImpl(appId, rmContext, conf, + null, user, null, ApplicationSubmissionContext.newInstance(appId, null, + queue, null, null, false, false, 0, amResource, null), null, null, 0, null, null, null); - rmContext.getRMApps().put(attId.getApplicationId(), rmApp); + rmContext.getRMApps().put(appId, rmApp); + RMAppEvent event = new RMAppEvent(appId, RMAppEventType.START); + resourceManager.getRMContext().getRMApps().get(appId).handle(event); + event = new RMAppEvent(appId, RMAppEventType.APP_NEW_SAVED); + resourceManager.getRMContext().getRMApps().get(appId).handle(event); + event = new RMAppEvent(appId, RMAppEventType.APP_ACCEPTED); + resourceManager.getRMContext().getRMApps().get(appId).handle(event); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( - attId.getApplicationId(), queue, user); + appId, queue, user); scheduler.handle(appAddedEvent); AppAttemptAddedSchedulerEvent attempAddedEvent = new AppAttemptAddedSchedulerEvent(attId, false); @@ -227,6 +245,9 @@ protected RMApp createMockRMApp(ApplicationAttemptId attemptId) { RMAppAttemptMetrics attemptMetric = mock(RMAppAttemptMetrics.class); when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric); when(app.getCurrentAppAttempt()).thenReturn(attempt); + ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class); + when(submissionContext.getUnmanagedAM()).thenReturn(false); + when(attempt.getSubmissionContext()).thenReturn(submissionContext); resourceManager.getRMContext().getRMApps() .put(attemptId.getApplicationId(), app); return app; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index e4583d1..a454801 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -1184,6 +1185,9 @@ private RMAppImpl createMockRMApp(ApplicationAttemptId attemptId, RMAppAttemptMetrics attemptMetric = mock(RMAppAttemptMetrics.class); when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric); when(app.getCurrentAppAttempt()).thenReturn(attempt); + ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class); + when(submissionContext.getUnmanagedAM()).thenReturn(false); + when(attempt.getSubmissionContext()).thenReturn(submissionContext); context.getRMApps().putIfAbsent(attemptId.getApplicationId(), app); return app; }