diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index 8b5d454..e550fb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -84,6 +84,7 @@ = new HashMap(); private List newlyAllocatedContainers = new ArrayList(); + private boolean isStopped = false; final Map> reservedContainers = new HashMap>(); @@ -119,6 +120,14 @@ public ApplicationId getApplicationId() { return appSchedulingInfo.getApplicationId(); } + public synchronized void updateBlacklist( + List blacklistAdditions, List blacklistRemovals) { + if (!isStopped) { + this.appSchedulingInfo.updateBlacklist( + blacklistAdditions, blacklistRemovals); + } + } + @Override public ApplicationAttemptId getApplicationAttemptId() { return appSchedulingInfo.getApplicationAttemptId(); @@ -138,7 +147,9 @@ public String getUser() { public synchronized void updateResourceRequests( List requests) { - this.appSchedulingInfo.updateResourceRequests(requests); + if (!this.isStopped) { + this.appSchedulingInfo.updateResourceRequests(requests); + } } public Map getResourceRequests(Priority priority) { @@ -165,6 +176,10 @@ public Resource getResource(Priority priority) { return appSchedulingInfo.getResource(priority); } + public boolean isBlacklisted(String resourceName) { + return this.appSchedulingInfo.isBlacklisted(resourceName); + } + /** * Is this application pending? * @return true if it is else false. @@ -188,6 +203,7 @@ public String getQueueName() { } public synchronized void stop(RMAppAttemptState rmAppAttemptFinalState) { + isStopped = true; // Cleanup all scheduling information appSchedulingInfo.stop(rmAppAttemptFinalState); } @@ -571,6 +587,11 @@ public synchronized NodeType getAllowedLocalityLevelByTime(Priority priority, synchronized public RMContainer allocate(NodeType type, FSSchedulerNode node, Priority priority, ResourceRequest request, Container container) { + + if (isStopped) { + return null; + } + // Update allowed locality level NodeType allowed = allowedLocalityLevel.get(priority); if (allowed != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index a3d5736..73cab79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -81,6 +81,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; @@ -882,7 +883,9 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, for (RMContainer container : application.getPreemptionContainers()) { preemptionContainerIds.add(container.getContainerId()); } - + + application.updateBlacklist(blacklistAdditions, blacklistRemovals); + return new Allocation(application.pullNewlyAllocatedContainers(), application.getHeadroom(), preemptionContainerIds); } @@ -1013,6 +1016,11 @@ public SchedulerNodeReport getNodeReport(NodeId nodeId) { FSSchedulerNode node = nodes.get(nodeId); return node == null ? null : new SchedulerNodeReport(node); } + + @Lock(Lock.NoLock.class) + FSSchedulerApp getApplication(ApplicationAttemptId applicationAttemptId) { + return applications.get(applicationAttemptId); + } public FSSchedulerApp getSchedulerApp(ApplicationAttemptId appAttemptId) { return applications.get(appAttemptId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index c69b431..37d745a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -82,11 +82,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.*; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; @@ -2431,4 +2427,35 @@ public void testDontAllowUndeclaredPools() throws Exception{ assertEquals(2, jerryQueue.getAppSchedulables().size()); assertEquals(2, defaultQueue.getAppSchedulables().size()); } + + @SuppressWarnings("resource") + @Test + public void testBlackListNodes() throws Exception { + final int GB = 1024; + resourceManager.start(); + FairScheduler fs = (FairScheduler) resourceManager.getResourceScheduler(); + + String host = "127.0.0.1"; + RMNode node = + MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host); + fs.handle(new NodeAddedSchedulerEvent(node)); + + ApplicationId appId = BuilderUtils.newApplicationId(100, 1); + ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + appId, 1); + SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "default", + "user"); + fs.handle(event); + + // Verify the blacklist can be updated independent of requesting containers + fs.allocate(appAttemptId, Collections.emptyList(), + Collections.emptyList(), + Collections.singletonList(host), null); + Assert.assertTrue(fs.getApplication(appAttemptId).isBlacklisted(host)); + fs.allocate(appAttemptId, Collections.emptyList(), + Collections.emptyList(), null, + Collections.singletonList(host)); + Assert.assertFalse(fs.getApplication(appAttemptId).isBlacklisted(host)); + resourceManager.stop(); + } }