diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index abbc2d7875f..0cdb4e34e23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -2010,11 +2010,19 @@ public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) { @Private private static final String AUTO_QUEUE_CREATION_V2_PREFIX = - "auto-queue-creation-v2"; + "auto-queue-creation-v2."; @Private public static final String AUTO_QUEUE_CREATION_V2_ENABLED = - AUTO_QUEUE_CREATION_V2_PREFIX + ".enabled"; + AUTO_QUEUE_CREATION_V2_PREFIX + "enabled"; + + @Private + public static final String AUTO_QUEUE_CREATION_V2_MAX_QUEUES = + AUTO_QUEUE_CREATION_V2_PREFIX + "max-queues"; + + @Private + public static final int + DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES = 1000; @Private public static final boolean DEFAULT_AUTO_QUEUE_CREATION_ENABLED = false; @@ -2130,6 +2138,28 @@ public int getAutoCreatedQueuesMaxChildQueuesLimit(String queuePath) { DEFAULT_AUTO_CREATE_QUEUE_MAX_QUEUES); } + /** + * Get the max number of queues that are allowed to be created under + * a parent queue which allowed auto creation v2. + * + * @param queuePath the parent queue's path + * @return the max number of queues allowed to be auto created, + * in new auto created. + */ + @Private + public int getAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath) { + return getInt(getQueuePrefix(queuePath) + + AUTO_QUEUE_CREATION_V2_MAX_QUEUES, + DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES); + } + + @VisibleForTesting + public void setAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath, + int maxQueues) { + setInt(getQueuePrefix(queuePath) + + AUTO_QUEUE_CREATION_V2_MAX_QUEUES, maxQueues); + } + @Private public static final String AUTO_CREATED_QUEUE_MANAGEMENT_POLICY = AUTO_CREATE_CHILD_QUEUE_PREFIX + "management-policy"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index b412e8a1dde..c0b23a2297d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -543,6 +543,16 @@ private CSQueue addDynamicChildQueue(String childQueuePath, boolean isLeaf) return queue; } + // Check if the max queues limit exceeds. + int maxQueues = csContext.getConfiguration(). + getAutoCreatedQueuesV2MaxChildQueuesLimit(getQueuePath()); + if (childQueues.size() >= maxQueues) { + throw new SchedulerDynamicEditException( + "Cannot auto create queue " + childQueuePath + ". Max Child " + + "Queue limit exceeded which is configured as : " + maxQueues + + " and number of child queues is : " + childQueues.size()); + } + // First, check if we allow creation or not boolean weightsAreUsed = false; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index d48bdc148f9..8cf4084d5e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -544,6 +544,27 @@ public void testAutoCreateQueueIfAmbiguousQueueNames() throws Exception { Assert.assertFalse(bAutoLeafQueue.hasChildQueues()); } + @Test + public void testAutoCreateQueueMaxQueuesLimit() throws Exception { + startScheduler(); + + csConf.setAutoCreatedQueuesV2MaxChildQueuesLimit("root.e", 5); + cs.reinitialize(csConf, mockRM.getRMContext()); + + for (int i = 0; i < 5; ++i) { + createQueue("root.e.q_" + i); + } + + // Check can't exceed max queues limit. + try { + createQueue("root.e.q_6"); + Assert.fail("Can't exceed max queues."); + } catch (Exception ex) { + Assert.assertTrue(ex + instanceof SchedulerDynamicEditException); + } + } + private LeafQueue createQueue(String queuePath) throws YarnException { return autoQueueHandler.autoCreateQueue( CSQueueUtils.extractQueuePath(queuePath));