diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 074e3711ba7..e7f6524fe69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -2040,10 +2040,18 @@ public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) { public static final String AUTO_QUEUE_CREATION_V2_MAX_QUEUES = AUTO_QUEUE_CREATION_V2_PREFIX + "max-queues"; + @Private + public static final String AUTO_QUEUE_CREATION_V2_MAX_DEPTH = + AUTO_QUEUE_CREATION_V2_PREFIX + "max-depth"; + @Private public static final int DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES = 1000; + @Private + public static final int + DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_DEPTH = 2; + @Private public static final boolean DEFAULT_AUTO_QUEUE_CREATION_ENABLED = false; @@ -2180,6 +2188,28 @@ public void setAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath, AUTO_QUEUE_CREATION_V2_MAX_QUEUES, maxQueues); } + /** + * Get the max depth that are allowed to be created under + * a parent queue which allowed auto creation v2. + * + * @param queuePath the parent queue's path + * @return the max depth of queues allowed to be auto created, + * in new auto created. + */ + @Private + public int getAutoCreatedQueuesV2MaxDepthLimit(String queuePath) { + return getInt(getQueuePrefix(queuePath) + + AUTO_QUEUE_CREATION_V2_MAX_DEPTH, + DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_DEPTH); + } + + @VisibleForTesting + public void setAutoCreatedQueuesV2MaxDepthLimit(String queuePath, + int maxDepth) { + setInt(getQueuePrefix(queuePath) + + AUTO_QUEUE_CREATION_V2_MAX_DEPTH, maxDepth); + } + @Private public static final String AUTO_CREATED_QUEUE_MANAGEMENT_POLICY = AUTO_CREATE_CHILD_QUEUE_PREFIX + "management-policy"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index a86a2536d05..28f010f7957 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -578,14 +578,6 @@ public LeafQueue createQueue(ApplicationPlacementContext queue) && parentCandidate.length() != 0) { ++firstStaticParentDistance; - if (firstStaticParentDistance > MAXIMUM_DYNAMIC_QUEUE_DEPTH) { - throw new SchedulerDynamicEditException( - "Could not auto create queue " + queue.getFullQueuePath() - + ". The distance of the LeafQueue from the first static " + - "ParentQueue is " + firstStaticParentDistance + ", which is " + - "above the limit."); - } - if (firstExistingParent == null) { parentsToCreate.addFirst(parentCandidate.toString()); } @@ -609,6 +601,14 @@ public LeafQueue createQueue(ApplicationPlacementContext queue) } ParentQueue existingParentQueue = (ParentQueue) firstExistingParent; + if (firstStaticParentDistance > + existingParentQueue.getMaxDepthForAutoQueueCreation()) { + throw new SchedulerDynamicEditException( + "Could not auto create queue " + queue.getFullQueuePath() + + ". The distance of the LeafQueue from the first static " + + "ParentQueue is " + firstStaticParentDistance + ", which is " + + "above the limit."); + } if (!existingParentQueue.isEligibleForAutoQueueCreation()) { throw new SchedulerDynamicEditException("Auto creation of queue " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 798c7103784..2480cc0a40a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -623,6 +623,16 @@ public boolean isEligibleForAutoQueueCreation() { return isDynamicQueue() || csContext.getConfiguration(). isAutoQueueCreationV2Enabled(getQueuePath()); } + + /** + * The max depth for v2 auto queue creation. + * The default max depth is 2. + * @return the max depth for this parent queue. + */ + public int getMaxDepthForAutoQueueCreation() { + return csContext.getConfiguration(). + getAutoCreatedQueuesV2MaxDepthLimit(getQueuePath()); + } @Override public void reinitialize(CSQueue newlyParsedQueue, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index b96c1e48968..8e0c3e89adc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -1137,6 +1137,39 @@ public void testRemoveDanglingAutoCreatedQueuesOnReinit() throws Exception { "when its dynamic parent is removed", bAutoLeaf); } + @Test + public void testAutoCreateQueueMaxDepthLimit() throws Exception { + startScheduler(); + + // Set max depth to 3 for root + csConf.setAutoCreatedQueuesV2MaxDepthLimit("root", 3); + cs.reinitialize(csConf, mockRM.getRMContext()); + + // Create 3 depth will pass. + createQueue("root.d1.d2.d3"); + + // Create 4 depth will fail. + try { + createQueue("root.e1.e2.e3.e4"); + Assert.fail("Can't exceed max queue depth."); + } catch (Exception ex) { + Assert.assertTrue(ex + instanceof SchedulerDynamicEditException); + } + + // Set max depth to 1 for root.d1 + csConf.setAutoCreatedQueuesV2MaxDepthLimit("root.d1", 1); + cs.reinitialize(csConf, mockRM.getRMContext()); + // Create 2 depth for root.d1 will fail. + try { + createQueue("root.d1.d_1.d_2"); + Assert.fail("Can't exceed max queue depth."); + } catch (Exception ex) { + Assert.assertTrue(ex + instanceof SchedulerDynamicEditException); + } + } + protected LeafQueue createQueue(String queuePath) throws YarnException, IOException { return autoQueueHandler.createQueue(