diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 1ace604..f8c4452 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -1472,6 +1472,7 @@ public void onReload(AllocationConfiguration queueInfo) { allocConf = queueInfo; allocConf.getDefaultSchedulingPolicy().initialize(clusterResource); queueMgr.updateAllocationConfiguration(allocConf); + maxRunningEnforcer.updateRunnabilityOnReloadAllocationConf(); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java index 2c90edd..4026492 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java @@ -17,12 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.PriorityQueue; +import java.util.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -198,7 +193,26 @@ public void updateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue) { } } } - + + public void updateRunnabilityOnReloadAllocationConf() { + List attempts = new ArrayList( + usersNonRunnableApps.values()); + Collections.sort(attempts, new Comparator() { + @Override + public int compare(FSAppAttempt o1, FSAppAttempt o2) { + return Long.compare(o1.getStartTime(), o2.getStartTime()); + } + }); + for (FSAppAttempt attempt : attempts) { + if (canAppBeRunnable(attempt.getQueue(), attempt.getUser())) { + trackRunnableApp(attempt); + attempt.getQueue().addApp(attempt, true); + attempt.getQueue().removeNonRunnableApp(attempt); + usersNonRunnableApps.remove(attempt.getUser(), attempt); + } + } + } + /** * Updates the relevant tracking variables after a runnable app with the given * queue and user has been removed. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index ac5748f..c490984 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -191,7 +191,29 @@ public void testMultiListStartTimeIteratorEmptyAppLists() { assertEquals(1, iter.next().getStartTime()); assertEquals(2, iter.next().getStartTime()); } - + + @Test + public void testUpdateAppRunnabilityWhenReloadingAllocationConf() { + FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1", true); + FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue2", true); + queueMaxApps.put("root", 2); + queueMaxApps.put("root.queue1", 1); + queueMaxApps.put("root.queue2", 1); + addApp(leaf1, "user"); + addApp(leaf2, "user"); + addApp(leaf2, "user"); + addApp(leaf2, "user"); + assertEquals(1, leaf1.getNumRunnableApps()); + assertEquals(1, leaf2.getNumRunnableApps()); + assertEquals(2, leaf2.getNumNonRunnableApps()); + queueMaxApps.put("root", 3); + queueMaxApps.put("root.queue2", 2); + maxAppsEnforcer.updateRunnabilityOnReloadAllocationConf(); + assertEquals(1, leaf1.getNumRunnableApps()); + assertEquals(2, leaf2.getNumRunnableApps()); + assertEquals(1, leaf2.getNumNonRunnableApps()); + } + private FSAppAttempt mockAppAttempt(long startTime) { FSAppAttempt schedApp = mock(FSAppAttempt.class); when(schedApp.getStartTime()).thenReturn(startTime);