diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index e562aaae5c3ffeca20812c37217cd08ceabb8cb2..b88f4db86526c0457f8cd4db9d641aa48deee24e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.records; +import java.util.List; import java.util.Map; import java.util.Set; @@ -454,30 +455,53 @@ public abstract void setKeepContainersAcrossApplicationAttempts( public abstract void setNodeLabelExpression(String nodeLabelExpression); /** - * Get ResourceRequest of AM container, if this is not null, scheduler will - * use this to acquire resource for AM container. + * Get the ResourceRequest of the AM container; if this is not null, + * scheduler will use this to acquire resource for AM container. * * If this is null, scheduler will assemble a ResourceRequest by using * getResource and getPriority of * ApplicationSubmissionContext. * - * Number of containers and Priority will be ignore. + * Number of containers and Priority will be ignored. * - * @return ResourceRequest of AM container + * @return ResourceRequest of the AM container */ @Public @Evolving public abstract ResourceRequest getAMContainerResourceRequest(); /** - * Set ResourceRequest of AM container - * @param request of AM container + * Set ResourceRequest of the AM container + * @param request of the AM container */ @Public @Evolving public abstract void setAMContainerResourceRequest(ResourceRequest request); /** + * Get the ResourceRequests of the AM container; if this is not null, + * scheduler will use this to acquire resource for AM container. + * + * If this is null, scheduler will use the ResourceRequest as determined by + * getAMContainerResourceRequest and its behavior. + * + * Number of containers and Priority will be ignored. + * + * @return List of ResourceRequests of the AM container + */ + @Public + @Evolving + public abstract List getAMContainerResourceRequests(); + + /** + * Set ResourceRequests of the AM container + * @param requests of the AM container + */ + @Public + @Evolving + public abstract void setAMContainerResourceRequests(List requests); + + /** * Get the attemptFailuresValidityInterval in milliseconds for the application * * @return the attemptFailuresValidityInterval diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index a8ba740949a535b70569bf416262b50163bed56e..eb34e1f104f8655e49b7b29de59ef8f487009683 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -326,6 +326,10 @@ message ExecutionTypeRequestProto { optional bool enforce_execution_type = 2 [default = false]; } +message ResourceRequestsProto { + repeated ResourceRequestProto resource_requests = 1; +} + enum AMCommandProto { AM_RESYNC = 1; AM_SHUTDOWN = 2; @@ -380,6 +384,7 @@ message ApplicationSubmissionContextProto { optional string node_label_expression = 16; optional ResourceRequestProto am_container_resource_request = 17; repeated ApplicationTimeoutMapProto application_timeouts = 18; + optional ResourceRequestsProto am_container_resource_requests = 19; } enum ApplicationTimeoutTypeProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index 62b54e7ed8e1307c20e5cde63061dac8784c7eae..cc6e4b021803556d347097b8296bb344d1018aab 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -48,6 +49,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestsProto; import com.google.common.base.CharMatcher; import com.google.protobuf.TextFormat; @@ -67,6 +69,7 @@ private Resource resource = null; private Set applicationTags = null; private ResourceRequest amResourceRequest = null; + private List amResourceRequests = null; private LogAggregationContext logAggregationContext = null; private ReservationId reservationId = null; private Map applicationTimeouts = null; @@ -131,6 +134,10 @@ private void mergeLocalToBuilder() { builder.setAmContainerResourceRequest( convertToProtoFormat(this.amResourceRequest)); } + if (this.amResourceRequests != null) { + builder.setAmContainerResourceRequests( + convertToProtoFormat(this.amResourceRequests)); + } if (this.logAggregationContext != null) { builder.setLogAggregationContext( convertToProtoFormat(this.logAggregationContext)); @@ -439,6 +446,22 @@ private ResourceRequestProto convertToProtoFormat(ResourceRequest t) { return ((ResourceRequestPBImpl)t).getProto(); } + private List convertFromProtoFormat(ResourceRequestsProto ps) { + List rs = new ArrayList<>(); + for (ResourceRequestProto p : ps.getResourceRequestsList()) { + rs.add(new ResourceRequestPBImpl(p)); + } + return rs; + } + + private ResourceRequestsProto convertToProtoFormat(List ts) { + ResourceRequestsProto.Builder builder = ResourceRequestsProto.newBuilder(); + for (ResourceRequest t : ts) { + builder.addResourceRequests(((ResourceRequestPBImpl)t).getProto()); + } + return builder.build(); + } + private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { return new ApplicationIdPBImpl(p); } @@ -498,6 +521,19 @@ public ResourceRequest getAMContainerResourceRequest() { } @Override + public List getAMContainerResourceRequests() { + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + if (this.amResourceRequests != null) { + return amResourceRequests; + } // Else via proto + if (!p.hasAmContainerResourceRequests()) { + return null; + } + amResourceRequests = convertFromProtoFormat(p.getAmContainerResourceRequests()); + return amResourceRequests; + } + + @Override public void setAMContainerResourceRequest(ResourceRequest request) { maybeInitBuilder(); if (request == null) { @@ -507,6 +543,15 @@ public void setAMContainerResourceRequest(ResourceRequest request) { } @Override + public void setAMContainerResourceRequests(List requests) { + maybeInitBuilder(); + if (requests == null) { + builder.clearAmContainerResourceRequests(); + } + this.amResourceRequests = requests; + } + + @Override public long getAttemptFailuresValidityInterval() { ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; return p.getAttemptFailuresValidityInterval(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index bc2195262d58ba36b9b209cbb3c1ca1d247c08aa..8a0560dafa88d6a55e4e10cef9c5308342b5989f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -19,7 +19,9 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.LinkedList; +import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -352,7 +354,7 @@ private RMAppImpl createAndPopulateNewRMApp( } ApplicationId applicationId = submissionContext.getApplicationId(); - ResourceRequest amReq = + List amReqs = validateAndCreateResourceRequest(submissionContext, isRecovery); // Verify and get the update application priority and set back to @@ -401,7 +403,7 @@ private RMAppImpl createAndPopulateNewRMApp( submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, submitTime, submissionContext.getApplicationType(), - submissionContext.getApplicationTags(), amReq, startTime); + submissionContext.getApplicationTags(), amReqs, startTime); // Concurrent app submissions with same applicationId will fail here // Concurrent app submissions with different applicationIds will not // influence each other @@ -427,7 +429,7 @@ private RMAppImpl createAndPopulateNewRMApp( return application; } - private ResourceRequest validateAndCreateResourceRequest( + private List validateAndCreateResourceRequest( ApplicationSubmissionContext submissionContext, boolean isRecovery) throws InvalidResourceRequestException { // Validation of the ApplicationSubmissionContext needs to be completed @@ -437,33 +439,41 @@ private ResourceRequest validateAndCreateResourceRequest( // Check whether AM resource requirements are within required limits if (!submissionContext.getUnmanagedAM()) { - ResourceRequest amReq = submissionContext.getAMContainerResourceRequest(); - if (amReq == null) { - amReq = BuilderUtils - .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, - ResourceRequest.ANY, submissionContext.getResource(), 1); + List amReqs = + submissionContext.getAMContainerResourceRequests(); + if (amReqs == null) { + ResourceRequest amReq = + submissionContext.getAMContainerResourceRequest(); + if (amReq == null) { + amReq = BuilderUtils + .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, + ResourceRequest.ANY, submissionContext.getResource(), 1); + } + amReqs = Collections.singletonList(amReq); } - // set label expression for AM container - if (null == amReq.getNodeLabelExpression()) { - amReq.setNodeLabelExpression(submissionContext - .getNodeLabelExpression()); + // set label expression for AM container and validate + for (ResourceRequest amReq : amReqs) { + if (null == amReq.getNodeLabelExpression()) { + amReq.setNodeLabelExpression(submissionContext + .getNodeLabelExpression()); + } + + try { + SchedulerUtils.normalizeAndValidateRequest(amReq, + scheduler.getMaximumResourceCapability(), + submissionContext.getQueue(), scheduler, isRecovery, rmContext); + } catch (InvalidResourceRequestException e) { + LOG.warn("RM app submission failed in validating AM resource request" + + " for application " + submissionContext.getApplicationId(), e); + throw e; + } + + scheduler.normalizeRequest(amReq); } - - try { - SchedulerUtils.normalizeAndValidateRequest(amReq, - scheduler.getMaximumResourceCapability(), - submissionContext.getQueue(), scheduler, isRecovery, rmContext); - } catch (InvalidResourceRequestException e) { - LOG.warn("RM app submission failed in validating AM resource request" - + " for application " + submissionContext.getApplicationId(), e); - throw e; - } - - scheduler.normalizeRequest(amReq); - return amReq; + return amReqs; } - + return null; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index b3a87a6e56011e2e37faf4b91dd58a1fce1a2632..43fd1fbfbdfd750790b89917a2272fddc1839a63 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Set; @@ -269,7 +270,7 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, ReservationId getReservationId(); - ResourceRequest getAMResourceRequest(); + List getAMResourceRequests(); Map getLogAggregationReportsForApp(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 4db595e25cdb78af2677953eabbe48e83da67e2e..18460a9edec5958fc4762a09d4d246778cf09cd5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -199,7 +199,7 @@ private RMAppEvent eventCausingFinalSaving; private RMAppState targetedFinalState; private RMAppState recoveredFinalState; - private ResourceRequest amReq; + private List amReqs; private CallerContext callerContext; @@ -427,10 +427,10 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, ApplicationSubmissionContext submissionContext, YarnScheduler scheduler, ApplicationMasterService masterService, long submitTime, String applicationType, Set applicationTags, - ResourceRequest amReq) { + List amReqs) { this(applicationId, rmContext, config, name, user, queue, submissionContext, scheduler, masterService, submitTime, applicationType, applicationTags, - amReq, -1); + amReqs, -1); } public RMAppImpl(ApplicationId applicationId, RMContext rmContext, @@ -438,7 +438,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, ApplicationSubmissionContext submissionContext, YarnScheduler scheduler, ApplicationMasterService masterService, long submitTime, String applicationType, Set applicationTags, - ResourceRequest amReq, long startTime) { + List amReqs, long startTime) { this.systemClock = SystemClock.getInstance(); @@ -461,7 +461,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, } this.applicationType = applicationType; this.applicationTags = applicationTags; - this.amReq = amReq; + this.amReqs = amReqs; if (submissionContext.getPriority() != null) { this.applicationPriority = Priority .newInstance(submissionContext.getPriority().getPriority()); @@ -1001,7 +1001,7 @@ private void createNewAttempt(ApplicationAttemptId appAttemptId) { // previously failed attempts(which should not include Preempted, // hardware error and NM resync) + 1) equal to the max-attempt // limit. - maxAppAttempts == (getNumFailedAppAttempts() + 1), amReq, + maxAppAttempts == (getNumFailedAppAttempts() + 1), amReqs, currentAMBlacklistManager); attempts.put(appAttemptId, attempt); currentAttempt = attempt; @@ -1695,8 +1695,8 @@ public ReservationId getReservationId() { } @Override - public ResourceRequest getAMResourceRequest() { - return this.amReq; + public List getAMResourceRequests() { + return this.amReqs; } protected Credentials parseCredentials() throws IOException { @@ -1976,7 +1976,9 @@ public String getAppNodeLabelExpression() { public String getAmNodeLabelExpression() { String amNodeLabelExpression = null; if (!getApplicationSubmissionContext().getUnmanagedAM()) { - amNodeLabelExpression = getAMResourceRequest().getNodeLabelExpression(); + amNodeLabelExpression = + getAMResourceRequests() != null && !getAMResourceRequests().isEmpty() + ? getAMResourceRequests().get(0).getNodeLabelExpression() : null; amNodeLabelExpression = (amNodeLabelExpression == null) ? NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET : amNodeLabelExpression; amNodeLabelExpression = (amNodeLabelExpression.trim().isEmpty()) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index f7ae48865dbf70a70378dba4bf4ca7846e9d657a..4477dced336af7dc21f48b92b36fb8e84e284389 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -192,7 +192,7 @@ private Object transitionTodo; private RMAppAttemptMetrics attemptMetrics = null; - private ResourceRequest amReq = null; + private List amReqs = null; private BlacklistManager blacklistedNodesForAM = null; private String amLaunchDiagnostics; @@ -485,17 +485,18 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, RMContext rmContext, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationSubmissionContext submissionContext, - Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq) { + Configuration conf, boolean maybeLastAttempt, + List amReqs) { this(appAttemptId, rmContext, scheduler, masterService, submissionContext, - conf, maybeLastAttempt, amReq, new DisabledBlacklistManager()); + conf, maybeLastAttempt, amReqs, new DisabledBlacklistManager()); } public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, RMContext rmContext, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationSubmissionContext submissionContext, - Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq, - BlacklistManager amBlacklistManager) { + Configuration conf, boolean maybeLastAttempt, + List amReqs, BlacklistManager amBlacklistManager) { this.conf = conf; this.applicationAttemptId = appAttemptId; this.rmContext = rmContext; @@ -515,7 +516,7 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, this.attemptMetrics = new RMAppAttemptMetrics(applicationAttemptId, rmContext); - this.amReq = amReq; + this.amReqs = amReqs; this.blacklistedNodesForAM = amBlacklistManager; } @@ -1046,14 +1047,13 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, // will be passed to scheduler, and scheduler will deduct the number after // AM container allocated - // Currently, following fields are all hard code, + // Currently, following fields are all hard coded, // TODO: change these fields when we want to support - // priority/resource-name/relax-locality specification for AM containers - // allocation. - appAttempt.amReq.setNumContainers(1); - appAttempt.amReq.setPriority(AM_CONTAINER_PRIORITY); - appAttempt.amReq.setResourceName(ResourceRequest.ANY); - appAttempt.amReq.setRelaxLocality(true); + // priority or multiple containers AM container allocation. + for (ResourceRequest amReq : appAttempt.amReqs) { + amReq.setNumContainers(1); + amReq.setPriority(AM_CONTAINER_PRIORITY); + } appAttempt.getAMBlacklistManager().refreshNodeHostCount( appAttempt.scheduler.getNumClusterNodes()); @@ -1069,7 +1069,7 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, Allocation amContainerAllocation = appAttempt.scheduler.allocate( appAttempt.applicationAttemptId, - Collections.singletonList(appAttempt.amReq), + appAttempt.amReqs, EMPTY_CONTAINER_RELEASE_LIST, amBlacklist.getBlacklistAdditions(), amBlacklist.getBlacklistRemovals(), null, null); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index b14bc204cc0dfab6e4f37678d94b1fbf7ff8db9d..7cf304bffa118372837ca2855016793cd157c41e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -138,18 +138,19 @@ public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId, Resource amResource; String partition; - if (rmApp == null || rmApp.getAMResourceRequest() == null) { + if (rmApp == null || rmApp.getAMResourceRequests() == null + || rmApp.getAMResourceRequests().isEmpty()) { // the rmApp may be undefined (the resource manager checks for this too) // and unmanaged applications do not provide an amResource request // in these cases, provide a default using the scheduler amResource = rmContext.getScheduler().getMinimumResourceCapability(); partition = CommonNodeLabelsManager.NO_LABEL; } else { - amResource = rmApp.getAMResourceRequest().getCapability(); + amResource = rmApp.getAMResourceRequests().get(0).getCapability(); partition = - (rmApp.getAMResourceRequest().getNodeLabelExpression() == null) + (rmApp.getAMResourceRequests().get(0).getNodeLabelExpression() == null) ? CommonNodeLabelsManager.NO_LABEL - : rmApp.getAMResourceRequest().getNodeLabelExpression(); + : rmApp.getAMResourceRequests().get(0).getNodeLabelExpression(); } setAppAMNodePartitionName(partition); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 4e85b671da70405453c8e69a76d39a916d595f6a..10e627a441052160090a6da83b7cb91562c431de 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -229,7 +229,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, appNodeLabelExpression = app.getApplicationSubmissionContext().getNodeLabelExpression(); amNodeLabelExpression = (unmanagedApplication) ? null - : app.getAMResourceRequest().getNodeLabelExpression(); + : app.getAMResourceRequests().get(0).getNodeLabelExpression(); // Setting partition based resource usage of application ResourceScheduler scheduler = rm.getRMContext().getScheduler(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 03cc3a0a63dbdabaeb8391da2e37c83f823b5f1d..e93d600a4e1b99adbfa1e7fcf1bd5fe501bd7a1a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -31,6 +31,8 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.concurrent.ConcurrentMap; @@ -70,6 +72,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; @@ -518,7 +521,53 @@ protected void setupDispatcher(RMContext rmContext, Configuration conf) { } @Test - public void testRMAppSubmit() throws Exception { + public void testRMAppSubmitAMContainerResourceRequests() throws Exception { + asContext.setResource(mockResource()); + asContext.setAMContainerResourceRequest( + ResourceRequest.newInstance(Priority.newInstance(2), + ResourceRequest.ANY, mockResource(), 1, true)); + List reqs = new ArrayList<>(); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(1), + ResourceRequest.ANY, mockResource(), 1, false)); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(1), + "/rack", mockResource(), 1, false)); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(1), + "/rack/node", mockResource(), 1, true)); + asContext.setAMContainerResourceRequests(reqs); + RMApp app = testRMAppSubmit(); + // setAMContainerResourceRequests has priority over + // setAMContainerResourceRequest and setResource + Assert.assertEquals(reqs, app.getAMResourceRequests()); + } + + @Test + public void testRMAppSubmitAMContainerResourceRequest() throws Exception { + asContext.setResource(mockResource()); + ResourceRequest req = + ResourceRequest.newInstance(Priority.newInstance(2), + ResourceRequest.ANY, mockResource(), 1, true); + asContext.setAMContainerResourceRequest(req); + asContext.setAMContainerResourceRequests(null); + RMApp app = testRMAppSubmit(); + // setAMContainerResourceRequest has priority over setResource + Assert.assertEquals(Collections.singletonList(req), + app.getAMResourceRequests()); + } + + @Test + public void testRMAppSubmitResource() throws Exception { + asContext.setResource(mockResource()); + asContext.setAMContainerResourceRequest(null); + asContext.setAMContainerResourceRequests(null); + RMApp app = testRMAppSubmit(); + // setResource + Assert.assertEquals(Collections.singletonList( + ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, + ResourceRequest.ANY, mockResource(), 1, true, "")), + app.getAMResourceRequests()); + } + + private RMApp testRMAppSubmit() throws Exception { appMonitor.submitApplication(asContext, "test"); RMApp app = rmContext.getRMApps().get(appId); Assert.assertNotNull("app is null", app); @@ -529,12 +578,14 @@ public void testRMAppSubmit() throws Exception { // wait for event to be processed int timeoutSecs = 0; - while ((getAppEventType() == RMAppEventType.KILL) && + while ((getAppEventType() == RMAppEventType.KILL) && timeoutSecs++ < 20) { Thread.sleep(1000); } Assert.assertEquals("app event type sent is wrong", RMAppEventType.START, getAppEventType()); + + return app; } @Test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index cb57f39534964eb5fc5bcc5705d6844a782927e8..eeac573b450194f780003e9688e645aecd35a181 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -35,6 +35,7 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -1048,9 +1049,9 @@ private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null, System.currentTimeMillis(), "YARN", null, - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - Resource.newInstance(1024, 1), 1)){ + Resource.newInstance(1024, 1), 1))){ @Override public ApplicationReport createAndGetApplicationReport( String clientUserName, boolean allowAccess) { @@ -1064,7 +1065,7 @@ public ApplicationReport createAndGetApplicationReport( return report; } }); - app.getAMResourceRequest().setNodeLabelExpression(amNodeLabelExpression); + app.getAMResourceRequests().get(0).setNodeLabelExpression(amNodeLabelExpression); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 9be52c6d3b6ca811b1aef2990455c62e327fb686..5246eb79a152d2b564c1fa9fca3dbdbb253c7ef0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -57,7 +57,7 @@ public abstract class MockAsm extends MockApps { public static class ApplicationBase implements RMApp { - ResourceRequest amReq; + List amReqs; @Override public String getUser() { throw new UnsupportedOperationException("Not supported yet."); @@ -204,8 +204,8 @@ public ReservationId getReservationId() { } @Override - public ResourceRequest getAMResourceRequest() { - return this.amReq; + public List getAMResourceRequests() { + return this.amReqs; } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 55e93c1e6cb40c52f0c71620429a37d15b1bc073..7005bca65853bfeefc27facc6a4ad978deba3873 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -526,7 +526,8 @@ private static RMApp createRMApp(ApplicationId appId) { when(app.getAppNodeLabelExpression()).thenCallRealMethod(); ResourceRequest amReq = mock(ResourceRequest.class); when(amReq.getNodeLabelExpression()).thenReturn("high-mem"); - when(app.getAMResourceRequest()).thenReturn(amReq); + when(app.getAMResourceRequests()) + .thenReturn(Collections.singletonList(amReq)); when(app.getAmNodeLabelExpression()).thenCallRealMethod(); when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10)); when(app.getCallerContext()) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 118b6bc74893ffa9e2a7f407a8eaea142adcd599..9290ff8faa0eb39c0128ee609094a15cc8bebbc4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -62,14 +63,14 @@ StringBuilder diagnostics = new StringBuilder(); RMAppAttempt attempt; int maxAppAttempts = 1; - ResourceRequest amReq; + List amReqs; public MockRMApp(int newid, long time, RMAppState newState) { finish = time; id = MockApps.newAppID(newid); state = newState; - amReq = ResourceRequest.newInstance(Priority.UNDEFINED, "0.0.0.0", - Resource.newInstance(0, 0), 1); + amReqs = Collections.singletonList(ResourceRequest.newInstance( + Priority.UNDEFINED, "0.0.0.0", Resource.newInstance(0, 0), 1)); } public MockRMApp(int newid, long time, RMAppState newState, String userName) { @@ -276,8 +277,8 @@ public ReservationId getReservationId() { } @Override - public ResourceRequest getAMResourceRequest() { - return this.amReq; + public List getAMResourceRequests() { + return this.amReqs; } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index c4459ad67c096ac7cd661223f01494c0164260f9..d98eb14ad6f0e602c26c3bfc89e56734a6862ad9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -30,8 +30,10 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Map; import org.apache.commons.logging.Log; @@ -271,7 +273,8 @@ protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN", null, mock(ResourceRequest.class)); + System.currentTimeMillis(), "YARN", null, + new ArrayList()); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), @@ -1024,9 +1027,9 @@ public void testRecoverApplication(ApplicationStateData appState, submissionContext.getQueue(), submissionContext, scheduler, null, appState.getSubmitTime(), submissionContext.getApplicationType(), submissionContext.getApplicationTags(), - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1)); + submissionContext.getResource(), 1))); Assert.assertEquals(RMAppState.NEW, application.getState()); RMAppEvent recoverEvent = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 497c6d0df8608cbe279681eacd41077d0334b184..885ee8e9ec05ae5c269053f7c69367f3a9a39b8e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -327,9 +327,9 @@ public void setUp() throws Exception { applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler, masterService, submissionContext, new Configuration(), false, - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1)); + submissionContext.getResource(), 1))); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); @@ -1107,9 +1107,9 @@ public void testLaunchedFailWhileAHSEnabled() { new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler,masterService, submissionContext, myConf, false, - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1)); + submissionContext.getResource(), 1))); //submit, schedule and allocate app attempt myApplicationAttempt.handle( @@ -1579,9 +1579,10 @@ public void testContainersCleanupForLastAttempt() { applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, new Configuration(), - true, BuilderUtils.newResourceRequest( + true, + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1)); + submissionContext.getResource(), 1))); when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true); when(submissionContext.getMaxAppAttempts()).thenReturn(1); @@ -1639,9 +1640,10 @@ public Allocation answer(InvocationOnMock invocation) applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, - new Configuration(), true, ResourceRequest.newInstance( + new Configuration(), true, + Collections.singletonList(ResourceRequest.newInstance( Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1), 3, - false, "label-expression")); + false, "label-expression"))); new RMAppAttemptImpl.ScheduleTransition().transition( (RMAppAttemptImpl) applicationAttempt, null); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 7382f3dffbacca52d4a644c42d13e244183ea4fc..5fd8f8bc26321a2bf6de0eb3a2d40ba44850ef87 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -622,7 +623,8 @@ public void testHeadroom() throws Exception { ResourceRequest amResourceRequest = mock(ResourceRequest.class); Resource amResource = Resources.createResource(0, 0); when(amResourceRequest.getCapability()).thenReturn(amResource); - when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest); + when(rmApp.getAMResourceRequests()).thenReturn( + Collections.singletonList(amResourceRequest)); Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any()); when(spyRMContext.getRMApps()).thenReturn(spyApps); RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index 1f87c533ff348ad5683b1f56705754703808be1d..3723c8b5f59280210d322a9b024830bda114ff6d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -642,7 +643,8 @@ public void testHeadroom() throws Exception { ResourceRequest amResourceRequest = mock(ResourceRequest.class); Resource amResource = Resources.createResource(0, 0); when(amResourceRequest.getCapability()).thenReturn(amResource); - when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest); + when(rmApp.getAMResourceRequests()).thenReturn( + Collections.singletonList(amResourceRequest)); Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any()); when(spyRMContext.getRMApps()).thenReturn(spyApps); RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 27860a6a30bcf066eb5806a9659d9a93e5b4e11f..ae59e3b4de05b500550c82611061a787e925a626 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -2931,7 +2931,7 @@ public void testAMUsedResource() throws Exception { RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName); assertEquals("RMApp does not containes minimum allocation", - minAllocResource, rmApp.getAMResourceRequest().getCapability()); + minAllocResource, rmApp.getAMResourceRequests().get(0).getCapability()); ResourceScheduler scheduler = rm.getRMContext().getScheduler(); LeafQueue queueA = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index e49532848c68250b9ab699aef2e8b3d953a74879..a5ae2d3c83d850a9eb11aeff77f30bbe53611344 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -153,7 +153,8 @@ private void setUpInternal(ResourceCalculator rC) throws Exception { amResourceRequest = mock(ResourceRequest.class); when(amResourceRequest.getCapability()).thenReturn( Resources.createResource(0, 0)); - when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest); + when(rmApp.getAMResourceRequests()).thenReturn( + Collections.singletonList(amResourceRequest)); Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any()); when(spyRMContext.getRMApps()).thenReturn(spyApps); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 5aa1e2db2c377030cf267bd54242c9072e27f1f4..f15d112760aae661223871a25e40447191453115 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -3210,6 +3210,84 @@ public void testCancelStrictLocality() throws IOException { assertEquals(1, app.getLiveContainers().size()); } + @Test + public void testAMStrictLocalityRack() throws IOException { + _testAMStrictLocality(false, false); + } + + @Test + public void testAMStrictLocalityNode() throws IOException { + _testAMStrictLocality(true, false); + } + + @Test + public void testAMStrictLocalityRackInvalid() throws IOException { + _testAMStrictLocality(false, true); + } + + @Test + public void testAMStrictLocalityNodeInvalid() throws IOException { + _testAMStrictLocality(true, true); + } + + private void _testAMStrictLocality(boolean node, boolean invalid) + throws IOException { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, + "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + RMNode node2 = MockNodes.newNodeInfo(2, Resources.createResource(1024), 2, + "127.0.0.2"); + NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); + scheduler.handle(nodeEvent2); + + List reqs = new ArrayList<>(); + ResourceRequest nodeRequest = createResourceRequest(1024, + node2.getHostName(), 1, 1, true); + if (node && invalid) { + nodeRequest.setResourceName("invalid"); + } + ResourceRequest rackRequest = createResourceRequest(1024, + node2.getRackName(), 1, 1, !node); + if (!node && invalid) { + rackRequest.setResourceName("invalid"); + } + ResourceRequest anyRequest = createResourceRequest(1024, + ResourceRequest.ANY, 1, 1, false); + reqs.add(anyRequest); + reqs.add(rackRequest); + if (node) { + reqs.add(nodeRequest); + } + + ApplicationAttemptId attId1 = + createSchedulingRequest("queue1", "user1", reqs); + + scheduler.update(); + + NodeUpdateSchedulerEvent node2UpdateEvent = + new NodeUpdateSchedulerEvent(node2); + + FSAppAttempt app = scheduler.getSchedulerApp(attId1); + + // node2 should get the container + scheduler.handle(node2UpdateEvent); + if (invalid) { + assertEquals(0, app.getLiveContainers().size()); + assertEquals(0, scheduler.getNode(node2.getNodeID()).getNumContainers()); + assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers()); + } else { + assertEquals(1, app.getLiveContainers().size()); + assertEquals(1, scheduler.getNode(node2.getNodeID()).getNumContainers()); + assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers()); + } + } + /** * If we update our ask to strictly request a node, it doesn't make sense to keep * a reservation on another. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 30f25e950e35ed7d0cf7e5b5b5c6c510a4011516..97067adec757a4a55bf36e9373e1690c3a84b1a8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1213,7 +1213,7 @@ public void testUnmarshalAppInfo() throws JSONException, Exception { assertEquals(app1.getApplicationId().toString(), appInfo.getAppId()); assertEquals(app1.getName(), appInfo.getName()); assertEquals(app1.createApplicationState(), appInfo.getState()); - assertEquals(app1.getAMResourceRequest().getCapability().getMemorySize(), + assertEquals(app1.getAMResourceRequests().get(0).getCapability().getMemorySize(), appInfo.getAllocatedMB()); rm.stop(); @@ -1427,7 +1427,7 @@ public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, expectedNumberOfElements++; appNodeLabelExpression = info.getString("appNodeLabelExpression"); } - if (app.getAMResourceRequest().getNodeLabelExpression() != null) { + if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) { expectedNumberOfElements++; amNodeLabelExpression = info.getString("amNodeLabelExpression"); } @@ -1534,7 +1534,7 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user, app.getApplicationSubmissionContext().getNodeLabelExpression(), appNodeLabelExpression); assertEquals("unmanagedApplication doesn't match", - app.getAMResourceRequest().getNodeLabelExpression(), + app.getAMResourceRequests().get(0).getNodeLabelExpression(), amNodeLabelExpression); assertEquals("amRPCAddress", AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()), @@ -1561,7 +1561,7 @@ public void verifyResourceRequestsGeneric(RMApp app, String nodeLabelExpression, int numContainers, boolean relaxLocality, int priority, String resourceName, long memory, long vCores, String executionType, boolean enforceExecutionType) { - ResourceRequest request = app.getAMResourceRequest(); + ResourceRequest request = app.getAMResourceRequests().get(0); assertEquals("nodeLabelExpression doesn't match", request.getNodeLabelExpression(), nodeLabelExpression); assertEquals("numContainers doesn't match", request.getNumContainers(),