diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 98fe5535cf300d84a82ddad8ab7418eb8264fce8..8132d890afa2203acca84f95e21e9c4426ad806c 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -22,6 +22,7 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -558,7 +559,8 @@ public ApplicationSubmissionContext createApplicationSubmissionContext(
amResourceRequest.setCapability(capability);
amResourceRequest.setNumContainers(1);
amResourceRequest.setNodeLabelExpression(amNodelabelExpression.trim());
- appContext.setAMContainerResourceRequest(amResourceRequest);
+ appContext.setAMContainerResourceRequests(
+ Collections.singletonList(amResourceRequest));
}
// set labels for the Job containers
appContext.setNodeLabelExpression(jobConf
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
index 279c8cec8535eab6bff0e1b355d79cfe77de6fcc..c2bda6235a0af6b26519608b79e9250d5207bc05 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
@@ -571,7 +571,7 @@ public void testNodeLabelExp() throws Exception {
buildSubmitContext(yarnRunner, jobConf);
assertEquals(appSubCtx.getNodeLabelExpression(), "GPU");
- assertEquals(appSubCtx.getAMContainerResourceRequest()
+ assertEquals(appSubCtx.getAMContainerResourceRequests().get(0)
.getNodeLabelExpression(), "highMem");
}
diff --git hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
index 5517362dc864f32282a0a44361fd3764f9271dd2..037c287644529d57d7ffc3e4cbe8299532e1e727 100644
--- hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
+++ hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
@@ -892,6 +892,11 @@ public int getNumClusterNodes() {
}
@Override
+ public int getNumClusterNodesByResourceName(String resourceName) {
+ return scheduler.getNumClusterNodesByResourceName(resourceName);
+ }
+
+ @Override
public SchedulerNodeReport getNodeReport(NodeId nodeId) {
return scheduler.getNodeReport(nodeId);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index e562aaae5c3ffeca20812c37217cd08ceabb8cb2..6fe9ec15c248668979faa2e6809727647631da0c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.api.records;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -100,7 +102,7 @@ public static ApplicationSubmissionContext newInstance(
amReq.setNumContainers(1);
amReq.setRelaxLocality(true);
amReq.setNodeLabelExpression(amContainerLabelExpression);
- context.setAMContainerResourceRequest(amReq);
+ context.setAMContainerResourceRequests(Collections.singletonList(amReq));
return context;
}
@@ -159,7 +161,8 @@ public static ApplicationSubmissionContext newInstance(
context.setApplicationType(applicationType);
context.setKeepContainersAcrossApplicationAttempts(keepContainers);
context.setNodeLabelExpression(appLabelExpression);
- context.setAMContainerResourceRequest(resourceRequest);
+ context.setAMContainerResourceRequests(
+ Collections.singletonList(resourceRequest));
return context;
}
@@ -454,30 +457,58 @@ public abstract void setKeepContainersAcrossApplicationAttempts(
public abstract void setNodeLabelExpression(String nodeLabelExpression);
/**
- * Get ResourceRequest of AM container, if this is not null, scheduler will
- * use this to acquire resource for AM container.
- *
+ * Get the ResourceRequest of the AM container; if this is not null,
+ * scheduler will use this to acquire resource for AM container.
+ *
* If this is null, scheduler will assemble a ResourceRequest by using
* getResource and getPriority of
* ApplicationSubmissionContext.
- *
- * Number of containers and Priority will be ignore.
- *
- * @return ResourceRequest of AM container
+ *
+ * Number of containers and Priority will be ignored.
+ *
+ * @return ResourceRequest of the AM container
+ * @deprecated See {@link #getAMContainerResourceRequests()}
*/
@Public
@Evolving
+ @Deprecated
public abstract ResourceRequest getAMContainerResourceRequest();
/**
- * Set ResourceRequest of AM container
- * @param request of AM container
+ * Set ResourceRequest of the AM container
+ * @param request of the AM container
+ * @deprecated See {@link #setAMContainerResourceRequests(List)}
*/
@Public
@Evolving
+ @Deprecated
public abstract void setAMContainerResourceRequest(ResourceRequest request);
/**
+ * Get the ResourceRequests of the AM container; if this is not null,
+ * scheduler will use this to acquire resource for AM container.
+ *
+ * If this is null, scheduler will use the ResourceRequest as determined by
+ * getAMContainerResourceRequest and its behavior.
+ *
+ * Number of containers and Priority will be ignored.
+ *
+ * @return List of ResourceRequests of the AM container
+ */
+ @Public
+ @Evolving
+ public abstract List getAMContainerResourceRequests();
+
+ /**
+ * Set ResourceRequests of the AM container.
+ * @param requests of the AM container
+ */
+ @Public
+ @Evolving
+ public abstract void setAMContainerResourceRequests(
+ List requests);
+
+ /**
* Get the attemptFailuresValidityInterval in milliseconds for the application
*
* @return the attemptFailuresValidityInterval
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index a6dbf3c2906db121d9a6b9fd6533895d4049408f..59f0c607fd8e50abecfbd03babe2f91815f096ff 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -378,7 +378,7 @@ message ApplicationSubmissionContextProto {
optional LogAggregationContextProto log_aggregation_context = 14;
optional ReservationIdProto reservation_id = 15;
optional string node_label_expression = 16;
- optional ResourceRequestProto am_container_resource_request = 17;
+ repeated ResourceRequestProto am_container_resource_request = 17;
repeated ApplicationTimeoutMapProto application_timeouts = 18;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
index 62b54e7ed8e1307c20e5cde63061dac8784c7eae..1a6719ad0959a59370c1f931b36681b0988d7905 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -66,7 +68,7 @@
private ContainerLaunchContext amContainer = null;
private Resource resource = null;
private Set applicationTags = null;
- private ResourceRequest amResourceRequest = null;
+ private List amResourceRequests = null;
private LogAggregationContext logAggregationContext = null;
private ReservationId reservationId = null;
private Map applicationTimeouts = null;
@@ -127,9 +129,10 @@ private void mergeLocalToBuilder() {
builder.clearApplicationTags();
builder.addAllApplicationTags(this.applicationTags);
}
- if (this.amResourceRequest != null) {
- builder.setAmContainerResourceRequest(
- convertToProtoFormat(this.amResourceRequest));
+ if (this.amResourceRequests != null) {
+ builder.clearAmContainerResourceRequest();
+ builder.addAllAmContainerResourceRequest(
+ convertToProtoFormat(this.amResourceRequests));
}
if (this.logAggregationContext != null) {
builder.setLogAggregationContext(
@@ -430,13 +433,23 @@ private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
private PriorityProto convertToProtoFormat(Priority t) {
return ((PriorityPBImpl)t).getProto();
}
-
- private ResourceRequestPBImpl convertFromProtoFormat(ResourceRequestProto p) {
- return new ResourceRequestPBImpl(p);
+
+ private List convertFromProtoFormat(
+ List ps) {
+ List rs = new ArrayList<>();
+ for (ResourceRequestProto p : ps) {
+ rs.add(new ResourceRequestPBImpl(p));
+ }
+ return rs;
}
- private ResourceRequestProto convertToProtoFormat(ResourceRequest t) {
- return ((ResourceRequestPBImpl)t).getProto();
+ private List convertToProtoFormat(
+ List ts) {
+ List rs = new ArrayList<>(ts.size());
+ for (ResourceRequest t : ts) {
+ rs.add(((ResourceRequestPBImpl)t).getProto());
+ }
+ return rs;
}
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
@@ -485,25 +498,46 @@ public void setNodeLabelExpression(String labelExpression) {
}
@Override
+ @Deprecated
public ResourceRequest getAMContainerResourceRequest() {
+ List reqs = getAMContainerResourceRequests();
+ if (reqs == null || reqs.isEmpty()) {
+ return null;
+ }
+ return getAMContainerResourceRequests().get(0);
+ }
+
+ @Override
+ public List getAMContainerResourceRequests() {
ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
- if (this.amResourceRequest != null) {
- return amResourceRequest;
+ if (this.amResourceRequests != null) {
+ return amResourceRequests;
} // Else via proto
- if (!p.hasAmContainerResourceRequest()) {
+ if (p.getAmContainerResourceRequestCount() == 0) {
return null;
}
- amResourceRequest = convertFromProtoFormat(p.getAmContainerResourceRequest());
- return amResourceRequest;
+ amResourceRequests =
+ convertFromProtoFormat(p.getAmContainerResourceRequestList());
+ return amResourceRequests;
}
@Override
+ @Deprecated
public void setAMContainerResourceRequest(ResourceRequest request) {
maybeInitBuilder();
if (request == null) {
builder.clearAmContainerResourceRequest();
}
- this.amResourceRequest = request;
+ this.amResourceRequests = Collections.singletonList(request);
+ }
+
+ @Override
+ public void setAMContainerResourceRequests(List requests) {
+ maybeInitBuilder();
+ if (requests == null) {
+ builder.clearAmContainerResourceRequest();
+ }
+ this.amResourceRequests = requests;
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index cc796e3f646274265bebaedf929e970783800395..33cd3a8b372fe8cc759920d764abba11cd828915 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -17,7 +17,9 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager;
+import java.util.Collections;
import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -31,6 +33,8 @@
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -338,7 +342,8 @@ protected void recoverApplication(ApplicationStateData appState,
// has been disabled. Reject the recovery of this application if it
// is true and give clear message so that user can react properly.
if (!appContext.getUnmanagedAM() &&
- application.getAMResourceRequest() == null &&
+ application.getAMResourceRequests() != null &&
+ !application.getAMResourceRequests().isEmpty() &&
!YarnConfiguration.areNodeLabelsEnabled(this.conf)) {
// check application submission context and see if am resource request
// or application itself contains any node label expression.
@@ -380,9 +385,9 @@ private RMAppImpl createAndPopulateNewRMApp(
}
ApplicationId applicationId = submissionContext.getApplicationId();
- ResourceRequest amReq = null;
+ List amReqs = null;
try {
- amReq = validateAndCreateResourceRequest(submissionContext, isRecovery);
+ amReqs = validateAndCreateResourceRequest(submissionContext, isRecovery);
} catch (InvalidLabelResourceRequestException e) {
// This can happen if the application had been submitted and run
// with Node Label enabled but recover with Node Label disabled.
@@ -445,7 +450,7 @@ private RMAppImpl createAndPopulateNewRMApp(
submissionContext.getQueue(),
submissionContext, this.scheduler, this.masterService,
submitTime, submissionContext.getApplicationType(),
- submissionContext.getApplicationTags(), amReq, startTime);
+ submissionContext.getApplicationTags(), amReqs, startTime);
// Concurrent app submissions with same applicationId will fail here
// Concurrent app submissions with different applicationIds will not
// influence each other
@@ -471,7 +476,7 @@ private RMAppImpl createAndPopulateNewRMApp(
return application;
}
- private ResourceRequest validateAndCreateResourceRequest(
+ private List validateAndCreateResourceRequest(
ApplicationSubmissionContext submissionContext, boolean isRecovery)
throws InvalidResourceRequestException {
// Validation of the ApplicationSubmissionContext needs to be completed
@@ -481,33 +486,71 @@ private ResourceRequest validateAndCreateResourceRequest(
// Check whether AM resource requirements are within required limits
if (!submissionContext.getUnmanagedAM()) {
- ResourceRequest amReq = submissionContext.getAMContainerResourceRequest();
- if (amReq == null) {
- amReq = BuilderUtils
- .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, submissionContext.getResource(), 1);
- }
-
- // set label expression for AM container
- if (null == amReq.getNodeLabelExpression()) {
- amReq.setNodeLabelExpression(submissionContext
- .getNodeLabelExpression());
+ List amReqs =
+ submissionContext.getAMContainerResourceRequests();
+ if (amReqs == null || amReqs.isEmpty()) {
+ if (submissionContext.getResource() != null) {
+ amReqs = Collections.singletonList(BuilderUtils
+ .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
+ ResourceRequest.ANY, submissionContext.getResource(), 1));
+ } else {
+ throw new InvalidResourceRequestException("Invalid resource request, "
+ + "no resources requested");
+ }
}
try {
- SchedulerUtils.normalizeAndValidateRequest(amReq,
- scheduler.getMaximumResourceCapability(),
- submissionContext.getQueue(), scheduler, isRecovery, rmContext);
+ // Find the ANY request and ensure there's only one
+ ResourceRequest anyReq = null;
+ for (ResourceRequest amReq : amReqs) {
+ if (amReq.getResourceName().equals(ResourceRequest.ANY)) {
+ if (anyReq == null) {
+ anyReq = amReq;
+ } else {
+ throw new InvalidResourceRequestException("Invalid resource "
+ + "request, only one resource request with "
+ + ResourceRequest.ANY + " is allowed");
+ }
+ }
+ }
+ if (anyReq == null) {
+ throw new InvalidResourceRequestException("Invalid resource request, "
+ + "no resource request specified with " + ResourceRequest.ANY);
+ }
+
+ // Make sure that all of the requests agree with the ANY request
+ // and have correct values
+ for (ResourceRequest amReq : amReqs) {
+ amReq.setCapability(anyReq.getCapability());
+ amReq.setExecutionTypeRequest(
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ amReq.setNumContainers(1);
+ amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY);
+ }
+
+ // set label expression for AM ANY request if not set
+ if (null == anyReq.getNodeLabelExpression()) {
+ anyReq.setNodeLabelExpression(submissionContext
+ .getNodeLabelExpression());
+ }
+
+ // Normalize all requests
+ for (ResourceRequest amReq : amReqs) {
+ SchedulerUtils.normalizeAndValidateRequest(amReq,
+ scheduler.getMaximumResourceCapability(),
+ submissionContext.getQueue(), scheduler, isRecovery, rmContext);
+
+ amReq.setCapability(
+ scheduler.getNormalizedResource(amReq.getCapability()));
+ }
+ return amReqs;
} catch (InvalidResourceRequestException e) {
LOG.warn("RM app submission failed in validating AM resource request"
+ " for application " + submissionContext.getApplicationId(), e);
throw e;
}
-
- amReq.setCapability(scheduler.getNormalizedResource(amReq.getCapability()));
- return amReq;
}
-
+
return null;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index b3a87a6e56011e2e37faf4b91dd58a1fce1a2632..43fd1fbfbdfd750790b89917a2272fddc1839a63 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import java.util.Collection;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -269,7 +270,7 @@ ApplicationReport createAndGetApplicationReport(String clientUserName,
ReservationId getReservationId();
- ResourceRequest getAMResourceRequest();
+ List getAMResourceRequests();
Map getLogAggregationReportsForApp();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 12ece3f6f3614374d3593dd683d543b2f2c2100b..8fa41b19d57fd1834b34b35a25a125396a7dc444 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -199,7 +199,7 @@
private RMAppEvent eventCausingFinalSaving;
private RMAppState targetedFinalState;
private RMAppState recoveredFinalState;
- private ResourceRequest amReq;
+ private List amReqs;
private CallerContext callerContext;
@@ -427,10 +427,10 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
ApplicationSubmissionContext submissionContext, YarnScheduler scheduler,
ApplicationMasterService masterService, long submitTime,
String applicationType, Set applicationTags,
- ResourceRequest amReq) {
+ List amReqs) {
this(applicationId, rmContext, config, name, user, queue, submissionContext,
scheduler, masterService, submitTime, applicationType, applicationTags,
- amReq, -1);
+ amReqs, -1);
}
public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
@@ -438,7 +438,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
ApplicationSubmissionContext submissionContext, YarnScheduler scheduler,
ApplicationMasterService masterService, long submitTime,
String applicationType, Set applicationTags,
- ResourceRequest amReq, long startTime) {
+ List amReqs, long startTime) {
this.systemClock = SystemClock.getInstance();
@@ -461,7 +461,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
}
this.applicationType = applicationType;
this.applicationTags = applicationTags;
- this.amReq = amReq;
+ this.amReqs = amReqs;
if (submissionContext.getPriority() != null) {
this.applicationPriority = Priority
.newInstance(submissionContext.getPriority().getPriority());
@@ -1001,7 +1001,7 @@ private void createNewAttempt(ApplicationAttemptId appAttemptId) {
// previously failed attempts(which should not include Preempted,
// hardware error and NM resync) + 1) equal to the max-attempt
// limit.
- maxAppAttempts == (getNumFailedAppAttempts() + 1), amReq,
+ maxAppAttempts == (getNumFailedAppAttempts() + 1), amReqs,
currentAMBlacklistManager);
attempts.put(appAttemptId, attempt);
currentAttempt = attempt;
@@ -1702,8 +1702,8 @@ public ReservationId getReservationId() {
}
@Override
- public ResourceRequest getAMResourceRequest() {
- return this.amReq;
+ public List getAMResourceRequests() {
+ return this.amReqs;
}
@Override
@@ -1971,7 +1971,9 @@ public String getAppNodeLabelExpression() {
public String getAmNodeLabelExpression() {
String amNodeLabelExpression = null;
if (!getApplicationSubmissionContext().getUnmanagedAM()) {
- amNodeLabelExpression = getAMResourceRequest().getNodeLabelExpression();
+ amNodeLabelExpression =
+ getAMResourceRequests() != null && !getAMResourceRequests().isEmpty()
+ ? getAMResourceRequests().get(0).getNodeLabelExpression() : null;
amNodeLabelExpression = (amNodeLabelExpression == null)
? NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET : amNodeLabelExpression;
amNodeLabelExpression = (amNodeLabelExpression.trim().isEmpty())
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index ab84985edcbc2a6ee34de90ec9de2c1edfbfcf1d..b3a6ebf7fbbc57e615df152b737877476d4e0ddd 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -193,7 +193,7 @@
private Object transitionTodo;
private RMAppAttemptMetrics attemptMetrics = null;
- private ResourceRequest amReq = null;
+ private List amReqs = null;
private BlacklistManager blacklistedNodesForAM = null;
private String amLaunchDiagnostics;
@@ -486,17 +486,18 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
ApplicationSubmissionContext submissionContext,
- Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq) {
+ Configuration conf, boolean maybeLastAttempt,
+ List amReqs) {
this(appAttemptId, rmContext, scheduler, masterService, submissionContext,
- conf, maybeLastAttempt, amReq, new DisabledBlacklistManager());
+ conf, maybeLastAttempt, amReqs, new DisabledBlacklistManager());
}
public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
ApplicationSubmissionContext submissionContext,
- Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq,
- BlacklistManager amBlacklistManager) {
+ Configuration conf, boolean maybeLastAttempt,
+ List amReqs, BlacklistManager amBlacklistManager) {
this.conf = conf;
this.applicationAttemptId = appAttemptId;
this.rmContext = rmContext;
@@ -516,7 +517,7 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
this.attemptMetrics =
new RMAppAttemptMetrics(applicationAttemptId, rmContext);
- this.amReq = amReq;
+ this.amReqs = amReqs;
this.blacklistedNodesForAM = amBlacklistManager;
}
@@ -1047,17 +1048,31 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt,
// will be passed to scheduler, and scheduler will deduct the number after
// AM container allocated
- // Currently, following fields are all hard code,
+ // Currently, following fields are all hard coded,
// TODO: change these fields when we want to support
- // priority/resource-name/relax-locality specification for AM containers
- // allocation.
- appAttempt.amReq.setNumContainers(1);
- appAttempt.amReq.setPriority(AM_CONTAINER_PRIORITY);
- appAttempt.amReq.setResourceName(ResourceRequest.ANY);
- appAttempt.amReq.setRelaxLocality(true);
+ // priority or multiple containers AM container allocation.
+ for (ResourceRequest amReq : appAttempt.amReqs) {
+ amReq.setNumContainers(1);
+ amReq.setPriority(AM_CONTAINER_PRIORITY);
+ }
- appAttempt.getAMBlacklistManager().refreshNodeHostCount(
- appAttempt.scheduler.getNumClusterNodes());
+ // If there's a strict locality requirement, set the number of nodes
+ // to consider for AM Blacklisting to the number of "eligible" nodes;
+ // otherwise, all nodes
+ int numNodes = -1;
+ for (ResourceRequest amReq : appAttempt.amReqs) {
+ if (amReq.getRelaxLocality()) {
+ numNodes = Math.max(numNodes, appAttempt.scheduler
+ .getNumClusterNodesByResourceName(amReq.getResourceName()));
+ }
+ }
+ if (numNodes == -1) {
+ numNodes = appAttempt.scheduler.getNumClusterNodes();
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting node count for blacklist to " + numNodes);
+ }
+ appAttempt.getAMBlacklistManager().refreshNodeHostCount(numNodes);
ResourceBlacklistRequest amBlacklist =
appAttempt.getAMBlacklistManager().getBlacklistUpdates();
@@ -1070,7 +1085,7 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt,
Allocation amContainerAllocation =
appAttempt.scheduler.allocate(
appAttempt.applicationAttemptId,
- Collections.singletonList(appAttempt.amReq),
+ appAttempt.amReqs,
EMPTY_CONTAINER_RELEASE_LIST,
amBlacklist.getBlacklistAdditions(),
amBlacklist.getBlacklistRemovals(),
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 64427b762ec0ab966d8f4eea5539cb0dafbc4842..971c9fa7b0f6e1baeffb25001e37f2084f440cf1 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -251,6 +251,16 @@ protected void initMaximumResourceCapability(Resource maximumAllocation) {
nodeTracker.setConfiguredMaxAllocation(maximumAllocation);
}
+ @Override
+ public int getNumClusterNodes() {
+ return nodeTracker.nodeCount();
+ }
+
+ @Override
+ public int getNumClusterNodesByResourceName(String resourceName) {
+ return nodeTracker.getNodesByResourceName(resourceName).size();
+ }
+
public SchedulerHealth getSchedulerHealth() {
return this.schedulerHealth;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 08e0603a84318a23909ef9c0eafee1de8c5b6ef0..000a7abb1ad0f76ef23821f3a6a8a84f90643bcb 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -124,6 +124,14 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues,
@Public
@Stable
public int getNumClusterNodes();
+
+ /**
+ * Get the number of nodes available in the cluster by resource name.
+ * @return the number of available nodes by resource name.
+ */
+ @LimitedPrivate("yarn")
+ @Evolving
+ public int getNumClusterNodesByResourceName(String resourceName);
/**
* The main api between the ApplicationMaster and the Scheduler.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 35177640293dff52dfa5f90a86ef6fdac662f8c1..751808323fd13484a7cd033125bf23e84a7c173d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -270,11 +270,6 @@ public void setResourceCalculator(ResourceCalculator rc) {
}
@Override
- public int getNumClusterNodes() {
- return nodeTracker.nodeCount();
- }
-
- @Override
public RMContext getRMContext() {
return this.rmContext;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 30b73052db586380255f1e090b23adbb78869192..a3b0f6395a3dfdac6fab48e949f9a14d2362b8a9 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -141,18 +141,20 @@ public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId,
Resource amResource;
String partition;
- if (rmApp == null || rmApp.getAMResourceRequest() == null) {
+ if (rmApp == null || rmApp.getAMResourceRequests() == null
+ || rmApp.getAMResourceRequests().isEmpty()) {
// the rmApp may be undefined (the resource manager checks for this too)
// and unmanaged applications do not provide an amResource request
// in these cases, provide a default using the scheduler
amResource = rmContext.getScheduler().getMinimumResourceCapability();
partition = CommonNodeLabelsManager.NO_LABEL;
} else {
- amResource = rmApp.getAMResourceRequest().getCapability();
+ amResource = rmApp.getAMResourceRequests().get(0).getCapability();
partition =
- (rmApp.getAMResourceRequest().getNodeLabelExpression() == null)
+ (rmApp.getAMResourceRequests().get(0)
+ .getNodeLabelExpression() == null)
? CommonNodeLabelsManager.NO_LABEL
- : rmApp.getAMResourceRequest().getNodeLabelExpression();
+ : rmApp.getAMResourceRequests().get(0).getNodeLabelExpression();
}
setAppAMNodePartitionName(partition);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 134efff7a0462d03ca0a5e19e566205614d0268b..524ac48c0378d745defddf10d0bff9a9678ec664 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1402,11 +1402,6 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues,
}
@Override
- public int getNumClusterNodes() {
- return nodeTracker.nodeCount();
- }
-
- @Override
public boolean checkAccess(UserGroupInformation callerUGI,
QueueACL acl, String queueName) {
try {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 64dbc7d200f726d48fba79b82a75eef1376b57e1..26c60483a2909ad7186224cf1593b7120906b217 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -306,11 +306,6 @@ public synchronized Configuration getConf() {
}
@Override
- public int getNumClusterNodes() {
- return nodeTracker.nodeCount();
- }
-
- @Override
public synchronized void setRMContext(RMContext rmContext) {
this.rmContext = rmContext;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 4e85b671da70405453c8e69a76d39a916d595f6a..10e627a441052160090a6da83b7cb91562c431de 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -229,7 +229,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
appNodeLabelExpression =
app.getApplicationSubmissionContext().getNodeLabelExpression();
amNodeLabelExpression = (unmanagedApplication) ? null
- : app.getAMResourceRequest().getNodeLabelExpression();
+ : app.getAMResourceRequests().get(0).getNodeLabelExpression();
// Setting partition based resource usage of application
ResourceScheduler scheduler = rm.getRMContext().getScheduler();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 7d19dab398f4d0990345d8859b6d4fa4a5dae40d..faf6a7ed61ad5e463a07006acbd1950afd106d08 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -24,6 +24,7 @@
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -678,6 +679,16 @@ public RMApp submitApp(Credentials cred, ByteBuffer tokensConf)
tokensConf);
}
+ public RMApp submitApp(List amResourceRequests) throws Exception {
+ return submitApp(amResourceRequests, "app1",
+ "user", null, false, null,
+ super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true,
+ false, false, null, 0, null, true,
+ amResourceRequests.get(0).getPriority(),
+ amResourceRequests.get(0).getNodeLabelExpression(), null, null);
+ }
+
public RMApp submitApp(Resource capability, String name, String user,
Map acls, boolean unmanaged, String queue,
int maxAppAttempts, Credentials ts, String appType,
@@ -688,6 +699,30 @@ public RMApp submitApp(Resource capability, String name, String user,
Map applicationTimeouts,
ByteBuffer tokensConf)
throws Exception {
+ priority = (priority == null) ? Priority.newInstance(0) : priority;
+ ResourceRequest amResourceRequest = ResourceRequest.newInstance(
+ priority, ResourceRequest.ANY, capability, 1);
+ if (amLabel != null && !amLabel.isEmpty()) {
+ amResourceRequest.setNodeLabelExpression(amLabel.trim());
+ }
+ return submitApp(Collections.singletonList(amResourceRequest), name, user,
+ acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted,
+ keepContainers, isAppIdProvided, applicationId,
+ attemptFailuresValidityInterval, logAggregationContext,
+ cancelTokensWhenComplete, priority, amLabel, applicationTimeouts,
+ tokensConf);
+ }
+
+ public RMApp submitApp(List amResourceRequests, String name,
+ String user, Map acls, boolean unmanaged,
+ String queue, int maxAppAttempts, Credentials ts, String appType,
+ boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
+ ApplicationId applicationId, long attemptFailuresValidityInterval,
+ LogAggregationContext logAggregationContext,
+ boolean cancelTokensWhenComplete, Priority priority, String amLabel,
+ Map applicationTimeouts,
+ ByteBuffer tokensConf)
+ throws Exception {
ApplicationId appId = isAppIdProvided ? applicationId : null;
ApplicationClientProtocol client = getClientRMService();
if (! isAppIdProvided) {
@@ -718,7 +753,6 @@ public RMApp submitApp(Resource capability, String name, String user,
sub.setApplicationType(appType);
ContainerLaunchContext clc = Records
.newRecord(ContainerLaunchContext.class);
- sub.setResource(capability);
clc.setApplicationACLs(acls);
if (ts != null && UserGroupInformation.isSecurityEnabled()) {
DataOutputBuffer dob = new DataOutputBuffer();
@@ -733,12 +767,12 @@ public RMApp submitApp(Resource capability, String name, String user,
sub.setLogAggregationContext(logAggregationContext);
}
sub.setCancelTokensWhenComplete(cancelTokensWhenComplete);
- ResourceRequest amResourceRequest = ResourceRequest.newInstance(
- Priority.newInstance(0), ResourceRequest.ANY, capability, 1);
if (amLabel != null && !amLabel.isEmpty()) {
- amResourceRequest.setNodeLabelExpression(amLabel.trim());
+ for (ResourceRequest amResourceRequest : amResourceRequests) {
+ amResourceRequest.setNodeLabelExpression(amLabel.trim());
+ }
}
- sub.setAMContainerResourceRequest(amResourceRequest);
+ sub.setAMContainerResourceRequests(amResourceRequests);
req.setApplicationSubmissionContext(sub);
UserGroupInformation fakeUser =
UserGroupInformation.createUserForTesting(user, new String[] {"someGroup"});
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 892f8ba36ec49adc0acdd68859b4887def3ca90a..24eb096ea2bb34375f25db5d14580519e6898b40 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -31,6 +31,8 @@
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
@@ -50,6 +52,8 @@
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -57,11 +61,13 @@
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
@@ -72,6 +78,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@@ -312,7 +319,7 @@ public void testQueueSubmitWithNoPermission() throws IOException {
ResourceRequest resReg =
ResourceRequest.newInstance(Priority.newInstance(0),
ResourceRequest.ANY, Resource.newInstance(1024, 1), 1);
- sub.setAMContainerResourceRequest(resReg);
+ sub.setAMContainerResourceRequests(Collections.singletonList(resReg));
req.setApplicationSubmissionContext(sub);
sub.setAMContainerSpec(mock(ContainerLaunchContext.class));
try {
@@ -522,8 +529,157 @@ protected void setupDispatcher(RMContext rmContext, Configuration conf) {
Assert.assertEquals("app event type is wrong before", RMAppEventType.KILL, appEventType);
}
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testRMAppSubmitAMContainerResourceRequests() throws Exception {
+ asContext.setResource(Resources.createResource(1024));
+ asContext.setAMContainerResourceRequest(
+ ResourceRequest.newInstance(Priority.newInstance(0),
+ ResourceRequest.ANY, Resources.createResource(1024), 1, true));
+ List reqs = new ArrayList<>();
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
+ ResourceRequest.ANY, Resources.createResource(1025), 1, false));
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
+ "/rack", Resources.createResource(1025), 1, false));
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
+ "/rack/node", Resources.createResource(1025), 1, true));
+ asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
+ // getAMContainerResourceRequest uses the first entry of
+ // getAMContainerResourceRequests
+ Assert.assertEquals(reqs.get(0), asContext.getAMContainerResourceRequest());
+ Assert.assertEquals(reqs, asContext.getAMContainerResourceRequests());
+ RMApp app = testRMAppSubmit();
+ for (ResourceRequest req : reqs) {
+ req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+ }
+ // setAMContainerResourceRequests has priority over
+ // setAMContainerResourceRequest and setResource
+ Assert.assertEquals(reqs, app.getAMResourceRequests());
+ }
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testRMAppSubmitAMContainerResourceRequest() throws Exception {
+ asContext.setResource(Resources.createResource(1024));
+ asContext.setAMContainerResourceRequests(null);
+ ResourceRequest req =
+ ResourceRequest.newInstance(Priority.newInstance(0),
+ ResourceRequest.ANY, Resources.createResource(1025), 1, true);
+ asContext.setAMContainerResourceRequest(cloneResourceRequest(req));
+ // getAMContainerResourceRequests uses a singleton list of
+ // getAMContainerResourceRequest
+ Assert.assertEquals(req, asContext.getAMContainerResourceRequest());
+ Assert.assertEquals(req, asContext.getAMContainerResourceRequests().get(0));
+ Assert.assertEquals(1, asContext.getAMContainerResourceRequests().size());
+ RMApp app = testRMAppSubmit();
+ req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+ // setAMContainerResourceRequest has priority over setResource
+ Assert.assertEquals(Collections.singletonList(req),
+ app.getAMResourceRequests());
+ }
+
@Test
- public void testRMAppSubmit() throws Exception {
+ public void testRMAppSubmitResource() throws Exception {
+ asContext.setResource(Resources.createResource(1024));
+ asContext.setAMContainerResourceRequests(null);
+ RMApp app = testRMAppSubmit();
+ // setResource
+ Assert.assertEquals(Collections.singletonList(
+ ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
+ ResourceRequest.ANY, Resources.createResource(1024), 1, true, "")),
+ app.getAMResourceRequests());
+ }
+
+ @Test
+ public void testRMAppSubmitNoResourceRequests() throws Exception {
+ asContext.setResource(null);
+ asContext.setAMContainerResourceRequests(null);
+ try {
+ testRMAppSubmit();
+ Assert.fail("Should have failed due to no ResourceRequest");
+ } catch (InvalidResourceRequestException e) {
+ Assert.assertEquals(
+ "Invalid resource request, no resources requested",
+ e.getMessage());
+ }
+ }
+
+ @Test
+ public void testRMAppSubmitAMContainerResourceRequestsDisagree()
+ throws Exception {
+ asContext.setResource(null);
+ List reqs = new ArrayList<>();
+ ResourceRequest anyReq = ResourceRequest.newInstance(
+ Priority.newInstance(1),
+ ResourceRequest.ANY, Resources.createResource(1024), 1, false, "label1",
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ reqs.add(anyReq);
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(2),
+ "/rack", Resources.createResource(1025), 2, false, "",
+ ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC)));
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(3),
+ "/rack/node", Resources.createResource(1026), 3, true, "",
+ ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC)));
+ asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
+ RMApp app = testRMAppSubmit();
+ // It should force the requests to all agree on these points
+ for (ResourceRequest req : reqs) {
+ req.setCapability(anyReq.getCapability());
+ req.setExecutionTypeRequest(
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ req.setNumContainers(1);
+ req.setPriority(Priority.newInstance(0));
+ }
+ Assert.assertEquals(reqs, app.getAMResourceRequests());
+ }
+
+ @Test
+ public void testRMAppSubmitAMContainerResourceRequestsNoAny()
+ throws Exception {
+ asContext.setResource(null);
+ List reqs = new ArrayList<>();
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(1),
+ "/rack", Resources.createResource(1025), 1, false));
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(1),
+ "/rack/node", Resources.createResource(1025), 1, true));
+ asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
+ // getAMContainerResourceRequest uses the first entry of
+ // getAMContainerResourceRequests
+ Assert.assertEquals(reqs, asContext.getAMContainerResourceRequests());
+ try {
+ testRMAppSubmit();
+ Assert.fail("Should have failed due to missing ANY ResourceRequest");
+ } catch (InvalidResourceRequestException e) {
+ Assert.assertEquals(
+ "Invalid resource request, no resource request specified with *",
+ e.getMessage());
+ }
+ }
+
+ @Test
+ public void testRMAppSubmitAMContainerResourceRequestsTwoManyAny()
+ throws Exception {
+ asContext.setResource(null);
+ List reqs = new ArrayList<>();
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(1),
+ ResourceRequest.ANY, Resources.createResource(1025), 1, false));
+ reqs.add(ResourceRequest.newInstance(Priority.newInstance(1),
+ ResourceRequest.ANY, Resources.createResource(1025), 1, false));
+ asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
+ // getAMContainerResourceRequest uses the first entry of
+ // getAMContainerResourceRequests
+ Assert.assertEquals(reqs, asContext.getAMContainerResourceRequests());
+ try {
+ testRMAppSubmit();
+ Assert.fail("Should have failed due to too many ANY ResourceRequests");
+ } catch (InvalidResourceRequestException e) {
+ Assert.assertEquals(
+ "Invalid resource request, only one resource request with * is " +
+ "allowed", e.getMessage());
+ }
+ }
+
+ private RMApp testRMAppSubmit() throws Exception {
appMonitor.submitApplication(asContext, "test");
RMApp app = rmContext.getRMApps().get(appId);
Assert.assertNotNull("app is null", app);
@@ -534,12 +690,14 @@ public void testRMAppSubmit() throws Exception {
// wait for event to be processed
int timeoutSecs = 0;
- while ((getAppEventType() == RMAppEventType.KILL) &&
+ while ((getAppEventType() == RMAppEventType.KILL) &&
timeoutSecs++ < 20) {
Thread.sleep(1000);
}
Assert.assertEquals("app event type sent is wrong", RMAppEventType.START,
getAppEventType());
+
+ return app;
}
@Test
@@ -737,6 +895,13 @@ private static ResourceScheduler mockResourceScheduler() {
ResourceCalculator rs = mock(ResourceCalculator.class);
when(scheduler.getResourceCalculator()).thenReturn(rs);
+ when(scheduler.getNormalizedResource(any())).thenAnswer(new Answer() {
+ @Override
+ public Resource answer(InvocationOnMock invocationOnMock) throws Throwable {
+ return (Resource) invocationOnMock.getArguments()[0];
+ }
+ });
+
return scheduler;
}
@@ -753,4 +918,26 @@ private static Resource mockResource() {
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
}
+ private static ResourceRequest cloneResourceRequest(ResourceRequest req) {
+ return ResourceRequest.newInstance(
+ Priority.newInstance(req.getPriority().getPriority()),
+ new String(req.getResourceName()),
+ Resource.newInstance(req.getCapability().getMemorySize(),
+ req.getCapability().getVirtualCores()),
+ req.getNumContainers(),
+ req.getRelaxLocality(),
+ req.getNodeLabelExpression() != null
+ ? new String(req.getNodeLabelExpression()) : null,
+ ExecutionTypeRequest.newInstance(
+ req.getExecutionTypeRequest().getExecutionType()));
+ }
+
+ private static List cloneResourceRequests(
+ List reqs) {
+ List cloneReqs = new ArrayList<>();
+ for (ResourceRequest req : reqs) {
+ cloneReqs.add(cloneResourceRequest(req));
+ }
+ return cloneReqs;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 637629059da3e43bd7d0ddbf76267069177f6d67..cda5df34eadd9642983e690bb733e04e301eca0e 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -38,6 +38,7 @@
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -1306,9 +1307,9 @@ private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler,
spy(new RMAppImpl(applicationId3, rmContext, config, null, null,
queueName, asContext, yarnScheduler, null,
System.currentTimeMillis(), "YARN", null,
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- Resource.newInstance(1024, 1), 1)){
+ Resource.newInstance(1024, 1), 1))){
@Override
public ApplicationReport createAndGetApplicationReport(
String clientUserName, boolean allowAccess) {
@@ -1322,7 +1323,8 @@ public ApplicationReport createAndGetApplicationReport(
return report;
}
});
- app.getAMResourceRequest().setNodeLabelExpression(amNodeLabelExpression);
+ app.getAMResourceRequests().get(0)
+ .setNodeLabelExpression(amNodeLabelExpression);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
ApplicationId.newInstance(123456, 1), 1);
RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId,
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
index c80a799e7d8b5da56ec2a224d49da4de30460a41..b4adf480b35fc0fed52a23f16e36e50089a2aafc 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -28,6 +29,9 @@
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -157,6 +161,186 @@ public void testNodeBlacklistingOnAMFailure() throws Exception {
}
@Test(timeout = 100000)
+ public void testNodeBlacklistingOnAMFailureStrictNodeLocality()
+ throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED,
+ true);
+
+ DrainDispatcher dispatcher = new DrainDispatcher();
+ MockRM rm = startRM(conf, dispatcher);
+ CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
+
+ // Register 5 nodes, so that we can blacklist atleast one if AM container
+ // is failed. As per calculation it will be like, 5nodes * 0.2 (default)=1.
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService());
+ nm1.registerNode();
+
+ MockNM nm2 =
+ new MockNM("127.0.0.2:2345", 8000, rm.getResourceTrackerService());
+ nm2.registerNode();
+
+ MockNM nm3 =
+ new MockNM("127.0.0.3:2345", 8000, rm.getResourceTrackerService());
+ nm3.registerNode();
+
+ MockNM nm4 =
+ new MockNM("127.0.0.4:2345", 8000, rm.getResourceTrackerService());
+ nm4.registerNode();
+
+ MockNM nm5 =
+ new MockNM("127.0.0.5:2345", 8000, rm.getResourceTrackerService());
+ nm5.registerNode();
+
+ // Specify a strict locality on nm2
+ List reqs = new ArrayList<>();
+ ResourceRequest nodeReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), nm2.getNodeId().getHost(),
+ Resource.newInstance(200, 1), 1, true);
+ ResourceRequest rackReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), "/default-rack",
+ Resource.newInstance(200, 1), 1, false);
+ ResourceRequest anyReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), ResourceRequest.ANY,
+ Resource.newInstance(200, 1), 1, false);
+ reqs.add(anyReq);
+ reqs.add(rackReq);
+ reqs.add(nodeReq);
+ RMApp app = rm.submitApp(reqs);
+
+ MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm2);
+ ContainerId amContainerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ RMContainer rmContainer = scheduler.getRMContainer(amContainerId);
+ NodeId nodeWhereAMRan = rmContainer.getAllocatedNode();
+ Assert.assertEquals(nm2.getNodeId(), nodeWhereAMRan);
+
+ // Set the exist status to INVALID so that we can verify that the system
+ // automatically blacklisting the node
+ makeAMContainerExit(rm, amContainerId, nm2, ContainerExitStatus.INVALID);
+
+ // restart the am
+ RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm);
+ System.out.println("New AppAttempt launched " + attempt.getAppAttemptId());
+
+ nm2.nodeHeartbeat(true);
+ dispatcher.await();
+
+ // Now the AM container should be allocated
+ MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
+
+ MockAM am2 = rm.sendAMLaunched(attempt.getAppAttemptId());
+ rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED);
+ amContainerId =
+ ContainerId.newContainerId(am2.getApplicationAttemptId(), 1);
+ rmContainer = scheduler.getRMContainer(amContainerId);
+ nodeWhereAMRan = rmContainer.getAllocatedNode();
+
+ // The second AM should be on the same node because the strict locality
+ // made the eligible nodes only 1, so the blacklisting threshold kicked in
+ System.out.println("AM ran on " + nodeWhereAMRan);
+ Assert.assertEquals(nm2.getNodeId(), nodeWhereAMRan);
+
+ am2.registerAppAttempt();
+ rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
+ }
+
+ @Test(timeout = 100000)
+ public void testNodeBlacklistingOnAMFailureRelaxedNodeLocality()
+ throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED,
+ true);
+
+ DrainDispatcher dispatcher = new DrainDispatcher();
+ MockRM rm = startRM(conf, dispatcher);
+ CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
+
+ // Register 5 nodes, so that we can blacklist atleast one if AM container
+ // is failed. As per calculation it will be like, 5nodes * 0.2 (default)=1.
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService());
+ nm1.registerNode();
+
+ MockNM nm2 =
+ new MockNM("127.0.0.2:2345", 8000, rm.getResourceTrackerService());
+ nm2.registerNode();
+
+ MockNM nm3 =
+ new MockNM("127.0.0.3:2345", 8000, rm.getResourceTrackerService());
+ nm3.registerNode();
+
+ MockNM nm4 =
+ new MockNM("127.0.0.4:2345", 8000, rm.getResourceTrackerService());
+ nm4.registerNode();
+
+ MockNM nm5 =
+ new MockNM("127.0.0.5:2345", 8000, rm.getResourceTrackerService());
+ nm5.registerNode();
+
+ // Specify a relaxed locality on nm2
+ List reqs = new ArrayList<>();
+ ResourceRequest nodeReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), nm2.getNodeId().getHost(),
+ Resource.newInstance(200, 1), 1, true);
+ ResourceRequest rackReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), "/default-rack",
+ Resource.newInstance(200, 1), 1, true);
+ ResourceRequest anyReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), ResourceRequest.ANY,
+ Resource.newInstance(200, 1), 1, true);
+ reqs.add(anyReq);
+ reqs.add(rackReq);
+ reqs.add(nodeReq);
+ RMApp app = rm.submitApp(reqs);
+
+ MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm2);
+ ContainerId amContainerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ RMContainer rmContainer = scheduler.getRMContainer(amContainerId);
+ NodeId nodeWhereAMRan = rmContainer.getAllocatedNode();
+ Assert.assertEquals(nm2.getNodeId(), nodeWhereAMRan);
+
+ // Set the exist status to INVALID so that we can verify that the system
+ // automatically blacklisting the node
+ makeAMContainerExit(rm, amContainerId, nm2, ContainerExitStatus.INVALID);
+
+ // restart the am
+ RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm);
+ System.out.println("New AppAttempt launched " + attempt.getAppAttemptId());
+
+ nm2.nodeHeartbeat(true);
+ nm1.nodeHeartbeat(true);
+ nm3.nodeHeartbeat(true);
+ nm4.nodeHeartbeat(true);
+ nm5.nodeHeartbeat(true);
+ dispatcher.await();
+
+ // Now the AM container should be allocated
+ MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
+
+ MockAM am2 = rm.sendAMLaunched(attempt.getAppAttemptId());
+ rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED);
+ amContainerId =
+ ContainerId.newContainerId(am2.getApplicationAttemptId(), 1);
+ rmContainer = scheduler.getRMContainer(amContainerId);
+ nodeWhereAMRan = rmContainer.getAllocatedNode();
+
+ // The second AM should be on a different node because the relaxed locality
+ // made the app schedulable on other nodes and nm2 is blacklisted
+ System.out.println("AM ran on " + nodeWhereAMRan);
+ Assert.assertNotEquals(nm2.getNodeId(), nodeWhereAMRan);
+
+ am2.registerAppAttempt();
+ rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
+ }
+
+ @Test(timeout = 100000)
public void testNoBlacklistingForNonSystemErrors() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 9be52c6d3b6ca811b1aef2990455c62e327fb686..5246eb79a152d2b564c1fa9fca3dbdbb253c7ef0 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -57,7 +57,7 @@
public abstract class MockAsm extends MockApps {
public static class ApplicationBase implements RMApp {
- ResourceRequest amReq;
+ List amReqs;
@Override
public String getUser() {
throw new UnsupportedOperationException("Not supported yet.");
@@ -204,8 +204,8 @@ public ReservationId getReservationId() {
}
@Override
- public ResourceRequest getAMResourceRequest() {
- return this.amReq;
+ public List getAMResourceRequests() {
+ return this.amReqs;
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 55e93c1e6cb40c52f0c71620429a37d15b1bc073..7005bca65853bfeefc27facc6a4ad978deba3873 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -526,7 +526,8 @@ private static RMApp createRMApp(ApplicationId appId) {
when(app.getAppNodeLabelExpression()).thenCallRealMethod();
ResourceRequest amReq = mock(ResourceRequest.class);
when(amReq.getNodeLabelExpression()).thenReturn("high-mem");
- when(app.getAMResourceRequest()).thenReturn(amReq);
+ when(app.getAMResourceRequests())
+ .thenReturn(Collections.singletonList(amReq));
when(app.getAmNodeLabelExpression()).thenCallRealMethod();
when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10));
when(app.getCallerContext())
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 118b6bc74893ffa9e2a7f407a8eaea142adcd599..9290ff8faa0eb39c0128ee609094a15cc8bebbc4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -21,6 +21,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -62,14 +63,14 @@
StringBuilder diagnostics = new StringBuilder();
RMAppAttempt attempt;
int maxAppAttempts = 1;
- ResourceRequest amReq;
+ List amReqs;
public MockRMApp(int newid, long time, RMAppState newState) {
finish = time;
id = MockApps.newAppID(newid);
state = newState;
- amReq = ResourceRequest.newInstance(Priority.UNDEFINED, "0.0.0.0",
- Resource.newInstance(0, 0), 1);
+ amReqs = Collections.singletonList(ResourceRequest.newInstance(
+ Priority.UNDEFINED, "0.0.0.0", Resource.newInstance(0, 0), 1));
}
public MockRMApp(int newid, long time, RMAppState newState, String userName) {
@@ -276,8 +277,8 @@ public ReservationId getReservationId() {
}
@Override
- public ResourceRequest getAMResourceRequest() {
- return this.amReq;
+ public List getAMResourceRequests() {
+ return this.amReqs;
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 488485112cb560bfba1ef79a1b2b5806ed0f85ca..5aa7af9e5199df48be945c7457f0b1846a6dead9 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -30,8 +30,10 @@
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -271,7 +273,8 @@ protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext)
submissionContext.setAMContainerSpec(mock(ContainerLaunchContext.class));
RMApp application = new RMAppImpl(applicationId, rmContext, conf, name,
user, queue, submissionContext, scheduler, masterService,
- System.currentTimeMillis(), "YARN", null, mock(ResourceRequest.class));
+ System.currentTimeMillis(), "YARN", null,
+ new ArrayList());
testAppStartState(applicationId, user, name, queue, application);
this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
@@ -1024,9 +1027,9 @@ public void testRecoverApplication(ApplicationStateData appState,
submissionContext.getQueue(), submissionContext, scheduler, null,
appState.getSubmitTime(), submissionContext.getApplicationType(),
submissionContext.getApplicationTags(),
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1)));
Assert.assertEquals(RMAppState.NEW, application.getState());
RMAppEvent recoverEvent =
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 786cc50801fcc87db952cde7c9036967fbc16980..933207e9e3566a9f193ff602dde6585c0d1248b8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -328,9 +328,9 @@ public void setUp() throws Exception {
applicationAttempt =
new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler,
masterService, submissionContext, new Configuration(), false,
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1)));
when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
when(application.getApplicationId()).thenReturn(applicationId);
@@ -1108,9 +1108,9 @@ public void testLaunchedFailWhileAHSEnabled() {
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
spyRMContext, scheduler,masterService,
submissionContext, myConf, false,
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1)));
//submit, schedule and allocate app attempt
myApplicationAttempt.handle(
@@ -1581,9 +1581,10 @@ public void testContainersCleanupForLastAttempt() {
applicationAttempt =
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext,
scheduler, masterService, submissionContext, new Configuration(),
- true, BuilderUtils.newResourceRequest(
+ true,
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1)));
when(submissionContext.getKeepContainersAcrossApplicationAttempts())
.thenReturn(true);
when(submissionContext.getMaxAppAttempts()).thenReturn(1);
@@ -1642,9 +1643,10 @@ public Allocation answer(InvocationOnMock invocation)
applicationAttempt =
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
spyRMContext, scheduler, masterService, submissionContext,
- new Configuration(), true, ResourceRequest.newInstance(
+ new Configuration(), true,
+ Collections.singletonList(ResourceRequest.newInstance(
Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1), 3,
- false, "label-expression"));
+ false, "label-expression")));
new RMAppAttemptImpl.ScheduleTransition().transition(
(RMAppAttemptImpl) applicationAttempt, null);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index bb0a1239fcae7cdff976921866094483c0c773dc..8aca235f8aa58bfe630c7f2cf1cbf81930e95947 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -28,6 +28,7 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -613,7 +614,8 @@ public void testHeadroom() throws Exception {
ResourceRequest amResourceRequest = mock(ResourceRequest.class);
Resource amResource = Resources.createResource(0, 0);
when(amResourceRequest.getCapability()).thenReturn(amResource);
- when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+ when(rmApp.getAMResourceRequests()).thenReturn(
+ Collections.singletonList(amResourceRequest));
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index b70a359f5913aa406b84157768c659416d537e4a..0aac2ef23daeb167de314fcbda479928216dbbcf 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -25,6 +25,7 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -639,7 +640,8 @@ public void testHeadroom() throws Exception {
ResourceRequest amResourceRequest = mock(ResourceRequest.class);
Resource amResource = Resources.createResource(0, 0);
when(amResourceRequest.getCapability()).thenReturn(amResource);
- when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+ when(rmApp.getAMResourceRequests()).thenReturn(
+ Collections.singletonList(amResourceRequest));
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 2b60ecfa6c311a8e0941cc074de8884526a032dc..4e7baf2f2665a86001cb7bb04f86cc4b3bb95374 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -3005,7 +3005,7 @@ public void testAMUsedResource() throws Exception {
RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName);
assertEquals("RMApp does not containes minimum allocation",
- minAllocResource, rmApp.getAMResourceRequest().getCapability());
+ minAllocResource, rmApp.getAMResourceRequests().get(0).getCapability());
ResourceScheduler scheduler = rm.getRMContext().getScheduler();
LeafQueue queueA =
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index ec1b84da6d82a92e7d89a540b96d87cca03b0621..1ea95386f3821ef0aa0c912d43771237c0420847 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -151,7 +151,8 @@ private void setUpInternal(ResourceCalculator rC) throws Exception {
amResourceRequest = mock(ResourceRequest.class);
when(amResourceRequest.getCapability()).thenReturn(
Resources.createResource(0, 0));
- when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+ when(rmApp.getAMResourceRequests()).thenReturn(
+ Collections.singletonList(amResourceRequest));
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index da5d3ad601393d0d50c4778b584641f14ea10127..eb416507b79fd22a8168eee3716e3c77c595e998 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3221,6 +3221,84 @@ public void testCancelStrictLocality() throws IOException {
assertEquals(1, app.getLiveContainers().size());
}
+ @Test
+ public void testAMStrictLocalityRack() throws IOException {
+ testAMStrictLocality(false, false);
+ }
+
+ @Test
+ public void testAMStrictLocalityNode() throws IOException {
+ testAMStrictLocality(true, false);
+ }
+
+ @Test
+ public void testAMStrictLocalityRackInvalid() throws IOException {
+ testAMStrictLocality(false, true);
+ }
+
+ @Test
+ public void testAMStrictLocalityNodeInvalid() throws IOException {
+ testAMStrictLocality(true, true);
+ }
+
+ private void testAMStrictLocality(boolean node, boolean invalid)
+ throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ RMNode node2 = MockNodes.newNodeInfo(2, Resources.createResource(1024), 2,
+ "127.0.0.2");
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+
+ List reqs = new ArrayList<>();
+ ResourceRequest nodeRequest = createResourceRequest(1024,
+ node2.getHostName(), 1, 1, true);
+ if (node && invalid) {
+ nodeRequest.setResourceName("invalid");
+ }
+ ResourceRequest rackRequest = createResourceRequest(1024,
+ node2.getRackName(), 1, 1, !node);
+ if (!node && invalid) {
+ rackRequest.setResourceName("invalid");
+ }
+ ResourceRequest anyRequest = createResourceRequest(1024,
+ ResourceRequest.ANY, 1, 1, false);
+ reqs.add(anyRequest);
+ reqs.add(rackRequest);
+ if (node) {
+ reqs.add(nodeRequest);
+ }
+
+ ApplicationAttemptId attId1 =
+ createSchedulingRequest("queue1", "user1", reqs);
+
+ scheduler.update();
+
+ NodeUpdateSchedulerEvent node2UpdateEvent =
+ new NodeUpdateSchedulerEvent(node2);
+
+ FSAppAttempt app = scheduler.getSchedulerApp(attId1);
+
+ // node2 should get the container
+ scheduler.handle(node2UpdateEvent);
+ if (invalid) {
+ assertEquals(0, app.getLiveContainers().size());
+ assertEquals(0, scheduler.getNode(node2.getNodeID()).getNumContainers());
+ assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ } else {
+ assertEquals(1, app.getLiveContainers().size());
+ assertEquals(1, scheduler.getNode(node2.getNodeID()).getNumContainers());
+ assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ }
+ }
+
/**
* If we update our ask to strictly request a node, it doesn't make sense to keep
* a reservation on another.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 30f25e950e35ed7d0cf7e5b5b5c6c510a4011516..fb9e8edfb0e7288876917582d9ed4a058ea46459 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1213,8 +1213,8 @@ public void testUnmarshalAppInfo() throws JSONException, Exception {
assertEquals(app1.getApplicationId().toString(), appInfo.getAppId());
assertEquals(app1.getName(), appInfo.getName());
assertEquals(app1.createApplicationState(), appInfo.getState());
- assertEquals(app1.getAMResourceRequest().getCapability().getMemorySize(),
- appInfo.getAllocatedMB());
+ assertEquals(app1.getAMResourceRequests().get(0).getCapability()
+ .getMemorySize(), appInfo.getAllocatedMB());
rm.stop();
}
@@ -1427,7 +1427,7 @@ public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
expectedNumberOfElements++;
appNodeLabelExpression = info.getString("appNodeLabelExpression");
}
- if (app.getAMResourceRequest().getNodeLabelExpression() != null) {
+ if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
expectedNumberOfElements++;
amNodeLabelExpression = info.getString("amNodeLabelExpression");
}
@@ -1534,7 +1534,7 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user,
app.getApplicationSubmissionContext().getNodeLabelExpression(),
appNodeLabelExpression);
assertEquals("unmanagedApplication doesn't match",
- app.getAMResourceRequest().getNodeLabelExpression(),
+ app.getAMResourceRequests().get(0).getNodeLabelExpression(),
amNodeLabelExpression);
assertEquals("amRPCAddress",
AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
@@ -1561,7 +1561,7 @@ public void verifyResourceRequestsGeneric(RMApp app,
String nodeLabelExpression, int numContainers, boolean relaxLocality,
int priority, String resourceName, long memory, long vCores,
String executionType, boolean enforceExecutionType) {
- ResourceRequest request = app.getAMResourceRequest();
+ ResourceRequest request = app.getAMResourceRequests().get(0);
assertEquals("nodeLabelExpression doesn't match",
request.getNodeLabelExpression(), nodeLabelExpression);
assertEquals("numContainers doesn't match", request.getNumContainers(),