diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index 00d1cda3088..397417c9df6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -27,8 +27,6 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.yarn.util.resource.ResourceUtils; -import org.apache.hadoop.yarn.util.resource.Resources; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -340,11 +338,14 @@ private void updateQueues(CSQueueStore existingQueues, } for (CSQueue queue : existingQueues.getQueues()) { - if (!((AbstractCSQueue) queue).isDynamicQueue() && newQueues.get( - queue.getQueuePath()) == null && !( - queue instanceof AutoCreatedLeafQueue && conf - .isAutoCreateChildQueueEnabled( - queue.getParent().getQueuePath()))) { + boolean isDanglingDynamicQueue = isDanglingDynamicQueue( + newQueues, existingQueues, queue); + boolean isRemovable = isDanglingDynamicQueue || !isDynamicQueue(queue) + && newQueues.get(queue.getQueuePath()) == null + && !(queue instanceof AutoCreatedLeafQueue && + conf.isAutoCreateChildQueueEnabled(queue.getParent().getQueuePath())); + + if (isRemovable) { existingQueues.remove(queue); } } @@ -435,4 +436,30 @@ public Priority getDefaultPriorityForQueue(String queueName) { getQueueStateManager() { return this.queueStateManager; } + + private boolean isDynamicQueue(CSQueue queue) { + return (queue instanceof AbstractCSQueue) && + ((AbstractCSQueue) queue).isDynamicQueue(); + } + + private boolean isDanglingDynamicQueue( + CSQueueStore newQueues, CSQueueStore existingQueues, + CSQueue queue) { + if (!isDynamicQueue(queue)) { + return false; + } + if (queue.getParent() == null) { + return true; + } + if (newQueues.get(queue.getParent().getQueuePath()) != null) { + return false; + } + CSQueue parent = existingQueues.get(queue.getParent().getQueuePath()); + + // A dynamic queue is dangling, if its parent is not parsed in newQueues + // or if its parent is not a dynamic queue. Dynamic queues are not parsed in + // newQueues but they are deleted automatically, so it is safe to assume + // that existingQueues contain valid dynamic queues. + return !isDynamicQueue(parent); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index c403d23f07b..eb41bc0c2a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -22,6 +22,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -968,6 +969,31 @@ public void testQueueInfoIfAmbiguousQueueNames() throws Exception { bAutoLeafQueue.getQueueInfo().getQueueName()); } + @Test + public void testRemoveDanglingAutoCreatedQueuesOnReinit() throws Exception { + startScheduler(); + + createQueue("root.a.a-auto"); + AbstractCSQueue aAuto = (AbstractCSQueue) cs. + getQueue("root.a.a-auto"); + Assert.assertTrue(aAuto.isDynamicQueue()); + + csConf.setState("root.a", QueueState.STOPPED); + cs.reinitialize(csConf, mockRM.getRMContext()); + aAuto = (AbstractCSQueue) cs. + getQueue("root.a.a-auto"); + Assert.assertEquals("root.a.a-auto is not in STOPPED state", QueueState.STOPPED, aAuto.getState()); + csConf.setQueues("root", new String[]{"b"}); + cs.reinitialize(csConf, mockRM.getRMContext()); + CSQueue aAutoNew = cs.getQueue("root.a.a-auto"); + Assert.assertNull(aAutoNew); + + submitApp(cs, USER0, "a-auto", "root.a"); + aAutoNew = cs.getQueue("root.a.a-auto"); + Assert.assertNotNull(aAutoNew); + + } + protected LeafQueue createQueue(String queuePath) throws YarnException { return autoQueueHandler.autoCreateQueue( CSQueueUtils.extractQueuePath(queuePath));