diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 04dbd2f..b3168e2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -26,6 +26,8 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.Set; +import java.util.TreeSet; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; @@ -314,19 +316,20 @@ public Resource assignContainer(FSSchedulerNode node) { return assigned; } - Comparator comparator = policy.getComparator(); - writeLock.lock(); - try { - Collections.sort(runnableApps, comparator); - } finally { - writeLock.unlock(); - } - // Release write lock here for better performance and avoiding deadlocks. - // runnableApps can be in unsorted state because of this section, - // but we can accept it in practice since the probability is low. + // Apps that have resource demands. + Set pendingForResourceApps = + new TreeSet(policy.getComparator()); readLock.lock(); try { - for (FSAppAttempt sched : runnableApps) { + for (FSAppAttempt app : runnableApps) { + // Either memory or vCores > 0, we believe it has resource request. + Resource pending = Resources.subtract(app.getDemand(), app.getResourceUsage()); + if (pending.getMemory() > Resources.none().getMemory() || + pending.getVirtualCores() > Resources.none().getVirtualCores()) { + pendingForResourceApps.add(app); + } + } + for (FSAppAttempt sched : pendingForResourceApps) { if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) { continue; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 69e0a8c..bb8ed3e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -3839,6 +3839,7 @@ public void testContinuousScheduling() throws Exception { ask.add(request); scheduler.stop(); scheduler.allocate(appAttemptId, ask, new ArrayList(), null, null); + scheduler.update(); scheduler.continuousSchedulingAttempt(); Assert.assertEquals(2048, app.getCurrentConsumption().getMemory()); Assert.assertEquals(2, app.getCurrentConsumption().getVirtualCores()); @@ -4137,6 +4138,7 @@ public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); scheduler.handle(nodeEvent); + scheduler.update(); scheduler.handle(updateEvent); scheduler.handle(updateEvent);