diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index f481de5..09d8ae5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueNotFoundException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.ContainersAndNMTokensAllocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; @@ -598,7 +599,7 @@ protected synchronized void addApplication(ApplicationId applicationId, } RMApp rmApp = rmContext.getRMApps().get(applicationId); - FSLeafQueue queue = assignToQueue(rmApp, queueName, user); + FSLeafQueue queue = assignToQueue(rmApp, queueName, user, isAppRecovering); if (queue == null) { return; } @@ -686,7 +687,8 @@ protected synchronized void addApplicationAttempt( * responsible to call the appropriate event-handler if the app is rejected. */ @VisibleForTesting - FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { + FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user, + boolean isAppRecovering) { FSLeafQueue queue = null; String appRejectMsg = null; @@ -694,6 +696,17 @@ FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy(); queueName = placementPolicy.assignAppToQueue(queueName, user); if (queueName == null) { + // During a restart, indicates QueuePlacementPolicy is changed + // not presently supported + if (isAppRecovering) { + String queueErrorMsg = + "Queue is missing during application recovery." + + " Queue removal during recovery is not presently supported by the" + + " fair scheduler, please restart with all queues configured" + + " which were present before shutdown/restart."; + LOG.fatal(queueErrorMsg); + throw new QueueNotFoundException(queueErrorMsg); + } appRejectMsg = "Application rejected by queue placement policy"; } else { queue = queueMgr.getLeafQueue(queueName, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 98877e7..43b6ab7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -86,6 +86,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueNotFoundException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; @@ -1038,8 +1039,10 @@ public void testAssignToQueue() throws Exception { RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); - FSLeafQueue queue1 = scheduler.assignToQueue(rmApp1, "default", "asterix"); - FSLeafQueue queue2 = scheduler.assignToQueue(rmApp2, "notdefault", "obelix"); + FSLeafQueue queue1 = + scheduler.assignToQueue(rmApp1, "default", "asterix", false); + FSLeafQueue queue2 = + scheduler.assignToQueue(rmApp2, "notdefault", "obelix", false); // assert FSLeafQueue's name is the correct name is the one set in the RMApp assertEquals(rmApp1.getQueue(), queue1.getName()); @@ -1062,8 +1065,9 @@ public void testAssignToNonLeafQueueReturnsNull() throws Exception { RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); // Trying to assign to non leaf queue would return null - assertNull(scheduler.assignToQueue(rmApp1, "root.child1", "tintin")); - assertNotNull(scheduler.assignToQueue(rmApp2, "root.child2", "snowy")); + assertNull(scheduler.assignToQueue(rmApp1, "root.child1", "tintin", false)); + assertNotNull(scheduler + .assignToQueue(rmApp2, "root.child2", "snowy", false)); } @Test @@ -1188,8 +1192,8 @@ public void testNestedUserQueue() throws IOException { scheduler.reinitialize(conf, resourceManager.getRMContext()); RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); - FSLeafQueue user1Leaf = scheduler.assignToQueue(rmApp1, "root.default", - "user1"); + FSLeafQueue user1Leaf = + scheduler.assignToQueue(rmApp1, "root.default", "user1", false); assertEquals("root.user1group.user1", user1Leaf.getName()); } @@ -4548,4 +4552,30 @@ public void testEmptyQueueNameInConfigFile() throws IOException { "Failed to initialize FairScheduler")); } } + + @Test(timeout = 10000, expected = QueueNotFoundException.class) + public void testAppRecoveringQueuePlacementPolicyChanged() throws IOException { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("2048mb,0vcores"); + out.println(""); + out.println("2048mb,0vcores"); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); + scheduler.assignToQueue(rmApp1, "root.QueueA", "user1", true); + } } -- 1.9.2.msysgit.0