reqs = new ArrayList<>();
+ ResourceRequest nodeReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), nm2.getNodeId().getHost(),
+ Resource.newInstance(200, 1), 1, true);
+ ResourceRequest rackReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), "/default-rack",
+ Resource.newInstance(200, 1), 1, true);
+ ResourceRequest anyReq = ResourceRequest.newInstance(
+ Priority.newInstance(0), ResourceRequest.ANY,
+ Resource.newInstance(200, 1), 1, true);
+ reqs.add(anyReq);
+ reqs.add(rackReq);
+ reqs.add(nodeReq);
+ RMApp app = rm.submitApp(reqs);
+
+ MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm2);
+ ContainerId amContainerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ RMContainer rmContainer = scheduler.getRMContainer(amContainerId);
+ NodeId nodeWhereAMRan = rmContainer.getAllocatedNode();
+ Assert.assertEquals(nm2.getNodeId(), nodeWhereAMRan);
+
+ // Set the exist status to INVALID so that we can verify that the system
+ // automatically blacklisting the node
+ makeAMContainerExit(rm, amContainerId, nm2, ContainerExitStatus.INVALID);
+
+ // restart the am
+ RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm);
+ System.out.println("New AppAttempt launched " + attempt.getAppAttemptId());
+
+ nm2.nodeHeartbeat(true);
+ nm1.nodeHeartbeat(true);
+ nm3.nodeHeartbeat(true);
+ nm4.nodeHeartbeat(true);
+ nm5.nodeHeartbeat(true);
+ dispatcher.await();
+
+ // Now the AM container should be allocated
+ MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
+
+ MockAM am2 = rm.sendAMLaunched(attempt.getAppAttemptId());
+ rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED);
+ amContainerId =
+ ContainerId.newContainerId(am2.getApplicationAttemptId(), 1);
+ rmContainer = scheduler.getRMContainer(amContainerId);
+ nodeWhereAMRan = rmContainer.getAllocatedNode();
+
+ // The second AM should be on a different node because the relaxed locality
+ // made the app schedulable on other nodes and nm2 is blacklisted
+ System.out.println("AM ran on " + nodeWhereAMRan);
+ Assert.assertNotEquals(nm2.getNodeId(), nodeWhereAMRan);
+
+ am2.registerAppAttempt();
+ rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
+ }
+
+ @Test(timeout = 100000)
public void testNoBlacklistingForNonSystemErrors() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
new file mode 100644
index 0000000000000000000000000000000000000000..5e3b404eda62451ab5b19b220f28d47c76740002
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TestRMServerUtils {
+ @Test
+ public void testGetApplicableNodeCountForAMLocality() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, false);
+ ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
+ Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100);
+ Mockito.when(
+ scheduler.getNumClusterNodesByResourceName("/rack1")).thenReturn(20);
+ Mockito.when(
+ scheduler.getNumClusterNodesByResourceName("node1")).thenReturn(1);
+ RMContext rmContext = Mockito.mock(RMContext.class);
+ Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
+
+ ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY,
+ true, null);
+ List reqs = new ArrayList<>();
+ reqs.add(anyReq);
+ Assert.assertEquals(100,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ ResourceRequest rackReq = createResourceRequest("/rack1", true, null);
+ reqs.add(rackReq);
+ Assert.assertEquals(20,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ anyReq.setRelaxLocality(false);
+ Assert.assertEquals(20,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(false);
+ Assert.assertEquals(100,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ ResourceRequest nodeReq = createResourceRequest("node1", false, null);
+ reqs.add(nodeReq);
+ Assert.assertEquals(100,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ nodeReq.setRelaxLocality(true);
+ Assert.assertEquals(1,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(true);
+ Assert.assertEquals(20,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ }
+
+ @Test
+ public void testGetApplicableNodeCountForAMLabels() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
+ Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100);
+ Mockito.when(
+ scheduler.getNumClusterNodesByResourceName("/rack1")).thenReturn(20);
+ Mockito.when(
+ scheduler.getNumClusterNodesByResourceName("node1")).thenReturn(1);
+ RMContext rmContext = Mockito.mock(RMContext.class);
+ Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
+ RMNodeLabelsManager labMan = Mockito.mock(RMNodeLabelsManager.class);
+ Mockito.when(rmContext.getNodeLabelManager()).thenReturn(labMan);
+ Mockito.when(labMan.getActiveNMCountPerLabel(
+ CommonNodeLabelsManager.NO_LABEL)).thenReturn(80);
+ Mockito.when(labMan.getActiveNMCountPerLabel("label1")).thenReturn(10);
+
+ ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY,
+ true, null);
+ List reqs = new ArrayList<>();
+ reqs.add(anyReq);
+ Assert.assertEquals(80,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ anyReq.setNodeLabelExpression("label1");
+ Assert.assertEquals(10,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ }
+
+ @Test
+ public void testGetApplicableNodeCountForAMLocalityAndLabels()
+ throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
+ Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100);
+ Mockito.when(
+ scheduler.getNumClusterNodesByResourceName("/rack1")).thenReturn(20);
+ Mockito.when(
+ scheduler.getNumClusterNodesByResourceName("node1")).thenReturn(1);
+ RMContext rmContext = Mockito.mock(RMContext.class);
+ Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
+ RMNodeLabelsManager labMan = Mockito.mock(RMNodeLabelsManager.class);
+ Mockito.when(rmContext.getNodeLabelManager()).thenReturn(labMan);
+ Mockito.when(labMan.getActiveNMCountPerLabel(
+ CommonNodeLabelsManager.NO_LABEL)).thenReturn(80);
+ Mockito.when(labMan.getActiveNMCountPerLabel("label1")).thenReturn(10);
+
+ ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY,
+ true, null);
+ List reqs = new ArrayList<>();
+ reqs.add(anyReq);
+ Assert.assertEquals(80,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ ResourceRequest rackReq = createResourceRequest("/rack1", true, null);
+ reqs.add(rackReq);
+ Assert.assertEquals(20,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ anyReq.setRelaxLocality(false);
+ Assert.assertEquals(20,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(false);
+ Assert.assertEquals(80,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ ResourceRequest nodeReq = createResourceRequest("node1", false, null);
+ reqs.add(nodeReq);
+ Assert.assertEquals(80,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ nodeReq.setRelaxLocality(true);
+ Assert.assertEquals(1,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(true);
+ Assert.assertEquals(20,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ anyReq.setNodeLabelExpression("label1");
+ rackReq.setNodeLabelExpression("label1");
+ nodeReq.setNodeLabelExpression("label1");
+ reqs = new ArrayList<>();
+ anyReq.setRelaxLocality(true);
+ reqs.add(anyReq);
+ Assert.assertEquals(10,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ rackReq.setRelaxLocality(true);
+ reqs.add(rackReq);
+ Assert.assertEquals(10,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ anyReq.setRelaxLocality(false);
+ Assert.assertEquals(10,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(false);
+ Assert.assertEquals(10,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+
+ nodeReq.setRelaxLocality(false);
+ reqs.add(nodeReq);
+ Assert.assertEquals(10,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ nodeReq.setRelaxLocality(true);
+ Assert.assertEquals(1,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ rackReq.setRelaxLocality(true);
+ Assert.assertEquals(10,
+ RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
+ }
+
+ private ResourceRequest createResourceRequest(String resource,
+ boolean relaxLocality, String nodeLabel) {
+ return ResourceRequest.newInstance(Priority.newInstance(0),
+ resource, Resource.newInstance(1, 1), 1, relaxLocality, nodeLabel);
+ }
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 9be52c6d3b6ca811b1aef2990455c62e327fb686..5246eb79a152d2b564c1fa9fca3dbdbb253c7ef0 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -57,7 +57,7 @@
public abstract class MockAsm extends MockApps {
public static class ApplicationBase implements RMApp {
- ResourceRequest amReq;
+ List amReqs;
@Override
public String getUser() {
throw new UnsupportedOperationException("Not supported yet.");
@@ -204,8 +204,8 @@ public ReservationId getReservationId() {
}
@Override
- public ResourceRequest getAMResourceRequest() {
- return this.amReq;
+ public List getAMResourceRequests() {
+ return this.amReqs;
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 55e93c1e6cb40c52f0c71620429a37d15b1bc073..7005bca65853bfeefc27facc6a4ad978deba3873 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -526,7 +526,8 @@ private static RMApp createRMApp(ApplicationId appId) {
when(app.getAppNodeLabelExpression()).thenCallRealMethod();
ResourceRequest amReq = mock(ResourceRequest.class);
when(amReq.getNodeLabelExpression()).thenReturn("high-mem");
- when(app.getAMResourceRequest()).thenReturn(amReq);
+ when(app.getAMResourceRequests())
+ .thenReturn(Collections.singletonList(amReq));
when(app.getAmNodeLabelExpression()).thenCallRealMethod();
when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10));
when(app.getCallerContext())
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 118b6bc74893ffa9e2a7f407a8eaea142adcd599..9290ff8faa0eb39c0128ee609094a15cc8bebbc4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -21,6 +21,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -62,14 +63,14 @@
StringBuilder diagnostics = new StringBuilder();
RMAppAttempt attempt;
int maxAppAttempts = 1;
- ResourceRequest amReq;
+ List amReqs;
public MockRMApp(int newid, long time, RMAppState newState) {
finish = time;
id = MockApps.newAppID(newid);
state = newState;
- amReq = ResourceRequest.newInstance(Priority.UNDEFINED, "0.0.0.0",
- Resource.newInstance(0, 0), 1);
+ amReqs = Collections.singletonList(ResourceRequest.newInstance(
+ Priority.UNDEFINED, "0.0.0.0", Resource.newInstance(0, 0), 1));
}
public MockRMApp(int newid, long time, RMAppState newState, String userName) {
@@ -276,8 +277,8 @@ public ReservationId getReservationId() {
}
@Override
- public ResourceRequest getAMResourceRequest() {
- return this.amReq;
+ public List getAMResourceRequests() {
+ return this.amReqs;
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 488485112cb560bfba1ef79a1b2b5806ed0f85ca..5aa7af9e5199df48be945c7457f0b1846a6dead9 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -30,8 +30,10 @@
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -271,7 +273,8 @@ protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext)
submissionContext.setAMContainerSpec(mock(ContainerLaunchContext.class));
RMApp application = new RMAppImpl(applicationId, rmContext, conf, name,
user, queue, submissionContext, scheduler, masterService,
- System.currentTimeMillis(), "YARN", null, mock(ResourceRequest.class));
+ System.currentTimeMillis(), "YARN", null,
+ new ArrayList());
testAppStartState(applicationId, user, name, queue, application);
this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
@@ -1024,9 +1027,9 @@ public void testRecoverApplication(ApplicationStateData appState,
submissionContext.getQueue(), submissionContext, scheduler, null,
appState.getSubmitTime(), submissionContext.getApplicationType(),
submissionContext.getApplicationTags(),
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1)));
Assert.assertEquals(RMAppState.NEW, application.getState());
RMAppEvent recoverEvent =
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 786cc50801fcc87db952cde7c9036967fbc16980..933207e9e3566a9f193ff602dde6585c0d1248b8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -328,9 +328,9 @@ public void setUp() throws Exception {
applicationAttempt =
new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler,
masterService, submissionContext, new Configuration(), false,
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1)));
when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
when(application.getApplicationId()).thenReturn(applicationId);
@@ -1108,9 +1108,9 @@ public void testLaunchedFailWhileAHSEnabled() {
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
spyRMContext, scheduler,masterService,
submissionContext, myConf, false,
- BuilderUtils.newResourceRequest(
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1)));
//submit, schedule and allocate app attempt
myApplicationAttempt.handle(
@@ -1581,9 +1581,10 @@ public void testContainersCleanupForLastAttempt() {
applicationAttempt =
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext,
scheduler, masterService, submissionContext, new Configuration(),
- true, BuilderUtils.newResourceRequest(
+ true,
+ Collections.singletonList(BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getResource(), 1));
+ submissionContext.getResource(), 1)));
when(submissionContext.getKeepContainersAcrossApplicationAttempts())
.thenReturn(true);
when(submissionContext.getMaxAppAttempts()).thenReturn(1);
@@ -1642,9 +1643,10 @@ public Allocation answer(InvocationOnMock invocation)
applicationAttempt =
new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(),
spyRMContext, scheduler, masterService, submissionContext,
- new Configuration(), true, ResourceRequest.newInstance(
+ new Configuration(), true,
+ Collections.singletonList(ResourceRequest.newInstance(
Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1), 3,
- false, "label-expression"));
+ false, "label-expression")));
new RMAppAttemptImpl.ScheduleTransition().transition(
(RMAppAttemptImpl) applicationAttempt, null);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index bb0a1239fcae7cdff976921866094483c0c773dc..8aca235f8aa58bfe630c7f2cf1cbf81930e95947 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -28,6 +28,7 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -613,7 +614,8 @@ public void testHeadroom() throws Exception {
ResourceRequest amResourceRequest = mock(ResourceRequest.class);
Resource amResource = Resources.createResource(0, 0);
when(amResourceRequest.getCapability()).thenReturn(amResource);
- when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+ when(rmApp.getAMResourceRequests()).thenReturn(
+ Collections.singletonList(amResourceRequest));
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index b70a359f5913aa406b84157768c659416d537e4a..0aac2ef23daeb167de314fcbda479928216dbbcf 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -25,6 +25,7 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -639,7 +640,8 @@ public void testHeadroom() throws Exception {
ResourceRequest amResourceRequest = mock(ResourceRequest.class);
Resource amResource = Resources.createResource(0, 0);
when(amResourceRequest.getCapability()).thenReturn(amResource);
- when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+ when(rmApp.getAMResourceRequests()).thenReturn(
+ Collections.singletonList(amResourceRequest));
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 2b60ecfa6c311a8e0941cc074de8884526a032dc..4e7baf2f2665a86001cb7bb04f86cc4b3bb95374 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -3005,7 +3005,7 @@ public void testAMUsedResource() throws Exception {
RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName);
assertEquals("RMApp does not containes minimum allocation",
- minAllocResource, rmApp.getAMResourceRequest().getCapability());
+ minAllocResource, rmApp.getAMResourceRequests().get(0).getCapability());
ResourceScheduler scheduler = rm.getRMContext().getScheduler();
LeafQueue queueA =
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index ec1b84da6d82a92e7d89a540b96d87cca03b0621..1ea95386f3821ef0aa0c912d43771237c0420847 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -151,7 +151,8 @@ private void setUpInternal(ResourceCalculator rC) throws Exception {
amResourceRequest = mock(ResourceRequest.class);
when(amResourceRequest.getCapability()).thenReturn(
Resources.createResource(0, 0));
- when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+ when(rmApp.getAMResourceRequests()).thenReturn(
+ Collections.singletonList(amResourceRequest));
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index da5d3ad601393d0d50c4778b584641f14ea10127..eb416507b79fd22a8168eee3716e3c77c595e998 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3221,6 +3221,84 @@ public void testCancelStrictLocality() throws IOException {
assertEquals(1, app.getLiveContainers().size());
}
+ @Test
+ public void testAMStrictLocalityRack() throws IOException {
+ testAMStrictLocality(false, false);
+ }
+
+ @Test
+ public void testAMStrictLocalityNode() throws IOException {
+ testAMStrictLocality(true, false);
+ }
+
+ @Test
+ public void testAMStrictLocalityRackInvalid() throws IOException {
+ testAMStrictLocality(false, true);
+ }
+
+ @Test
+ public void testAMStrictLocalityNodeInvalid() throws IOException {
+ testAMStrictLocality(true, true);
+ }
+
+ private void testAMStrictLocality(boolean node, boolean invalid)
+ throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ RMNode node2 = MockNodes.newNodeInfo(2, Resources.createResource(1024), 2,
+ "127.0.0.2");
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+
+ List reqs = new ArrayList<>();
+ ResourceRequest nodeRequest = createResourceRequest(1024,
+ node2.getHostName(), 1, 1, true);
+ if (node && invalid) {
+ nodeRequest.setResourceName("invalid");
+ }
+ ResourceRequest rackRequest = createResourceRequest(1024,
+ node2.getRackName(), 1, 1, !node);
+ if (!node && invalid) {
+ rackRequest.setResourceName("invalid");
+ }
+ ResourceRequest anyRequest = createResourceRequest(1024,
+ ResourceRequest.ANY, 1, 1, false);
+ reqs.add(anyRequest);
+ reqs.add(rackRequest);
+ if (node) {
+ reqs.add(nodeRequest);
+ }
+
+ ApplicationAttemptId attId1 =
+ createSchedulingRequest("queue1", "user1", reqs);
+
+ scheduler.update();
+
+ NodeUpdateSchedulerEvent node2UpdateEvent =
+ new NodeUpdateSchedulerEvent(node2);
+
+ FSAppAttempt app = scheduler.getSchedulerApp(attId1);
+
+ // node2 should get the container
+ scheduler.handle(node2UpdateEvent);
+ if (invalid) {
+ assertEquals(0, app.getLiveContainers().size());
+ assertEquals(0, scheduler.getNode(node2.getNodeID()).getNumContainers());
+ assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ } else {
+ assertEquals(1, app.getLiveContainers().size());
+ assertEquals(1, scheduler.getNode(node2.getNodeID()).getNumContainers());
+ assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ }
+ }
+
/**
* If we update our ask to strictly request a node, it doesn't make sense to keep
* a reservation on another.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 30f25e950e35ed7d0cf7e5b5b5c6c510a4011516..fb9e8edfb0e7288876917582d9ed4a058ea46459 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1213,8 +1213,8 @@ public void testUnmarshalAppInfo() throws JSONException, Exception {
assertEquals(app1.getApplicationId().toString(), appInfo.getAppId());
assertEquals(app1.getName(), appInfo.getName());
assertEquals(app1.createApplicationState(), appInfo.getState());
- assertEquals(app1.getAMResourceRequest().getCapability().getMemorySize(),
- appInfo.getAllocatedMB());
+ assertEquals(app1.getAMResourceRequests().get(0).getCapability()
+ .getMemorySize(), appInfo.getAllocatedMB());
rm.stop();
}
@@ -1427,7 +1427,7 @@ public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
expectedNumberOfElements++;
appNodeLabelExpression = info.getString("appNodeLabelExpression");
}
- if (app.getAMResourceRequest().getNodeLabelExpression() != null) {
+ if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
expectedNumberOfElements++;
amNodeLabelExpression = info.getString("amNodeLabelExpression");
}
@@ -1534,7 +1534,7 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user,
app.getApplicationSubmissionContext().getNodeLabelExpression(),
appNodeLabelExpression);
assertEquals("unmanagedApplication doesn't match",
- app.getAMResourceRequest().getNodeLabelExpression(),
+ app.getAMResourceRequests().get(0).getNodeLabelExpression(),
amNodeLabelExpression);
assertEquals("amRPCAddress",
AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
@@ -1561,7 +1561,7 @@ public void verifyResourceRequestsGeneric(RMApp app,
String nodeLabelExpression, int numContainers, boolean relaxLocality,
int priority, String resourceName, long memory, long vCores,
String executionType, boolean enforceExecutionType) {
- ResourceRequest request = app.getAMResourceRequest();
+ ResourceRequest request = app.getAMResourceRequests().get(0);
assertEquals("nodeLabelExpression doesn't match",
request.getNodeLabelExpression(), nodeLabelExpression);
assertEquals("numContainers doesn't match", request.getNumContainers(),