diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java index e847737388c..02b5cfd7547 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java @@ -31,7 +31,6 @@ */ public class CapacitySchedulerAutoQueueHandler { private final CapacitySchedulerQueueManager queueManager; - private static final int MAXIMUM_DEPTH_ALLOWED = 2; public CapacitySchedulerAutoQueueHandler( CapacitySchedulerQueueManager queueManager) { @@ -114,7 +113,7 @@ public LeafQueue autoCreateQueue(ApplicationPlacementContext queue) private int extractDepthLimit(ParentQueue parentQueue) { if (parentQueue.isEligibleForAutoQueueCreation()) { - return MAXIMUM_DEPTH_ALLOWED; + return parentQueue.getMaxDepthForAutoQueueCreation(); } else { return 0; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 08b38a1707e..91758de6166 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -2020,10 +2020,18 @@ public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) { public static final String AUTO_QUEUE_CREATION_V2_MAX_QUEUES = AUTO_QUEUE_CREATION_V2_PREFIX + "max-queues"; + @Private + public static final String AUTO_QUEUE_CREATION_V2_MAX_DEPTH = + AUTO_QUEUE_CREATION_V2_PREFIX + "max-depth"; + @Private public static final int DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES = 1000; + @Private + public static final int + DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_DEPTH = 2; + @Private public static final boolean DEFAULT_AUTO_QUEUE_CREATION_ENABLED = false; @@ -2160,6 +2168,28 @@ public void setAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath, AUTO_QUEUE_CREATION_V2_MAX_QUEUES, maxQueues); } + /** + * Get the max depth that are allowed to be created under + * a parent queue which allowed auto creation v2. + * + * @param queuePath the parent queue's path + * @return the max depth of queues allowed to be auto created, + * in new auto created. + */ + @Private + public int getAutoCreatedQueuesV2MaxDepthLimit(String queuePath) { + return getInt(getQueuePrefix(queuePath) + + AUTO_QUEUE_CREATION_V2_MAX_DEPTH, + DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_DEPTH); + } + + @VisibleForTesting + public void setAutoCreatedQueuesV2MaxDepthLimit(String queuePath, + int maxDepth) { + setInt(getQueuePrefix(queuePath) + + AUTO_QUEUE_CREATION_V2_MAX_DEPTH, maxDepth); + } + @Private public static final String AUTO_CREATED_QUEUE_MANAGEMENT_POLICY = AUTO_CREATE_CHILD_QUEUE_PREFIX + "management-policy"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index f79ffcbb33c..55df737fd0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.StringUtils; @@ -593,6 +594,15 @@ public boolean isEligibleForAutoQueueCreation() { return isDynamicQueue() || csContext.getConfiguration(). isAutoQueueCreationV2Enabled(getQueuePath()); } + + /** + * The max depth for v2 auto queue creation. + * The default max depth is 2. + */ + public int getMaxDepthForAutoQueueCreation() { + return csContext.getConfiguration(). + getAutoCreatedQueuesV2MaxDepthLimit(getQueuePath()); + } @Override public void reinitialize(CSQueue newlyParsedQueue, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index 4facf943b6e..d9c1000446c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -565,6 +565,39 @@ public void testAutoCreateQueueMaxQueuesLimit() throws Exception { } } + @Test + public void testAutoCreateQueueMaxDepthLimit() throws Exception { + startScheduler(); + + // Set max depth to 3 for root + csConf.setAutoCreatedQueuesV2MaxDepthLimit("root", 3); + cs.reinitialize(csConf, mockRM.getRMContext()); + + // Create 3 depth will pass. + createQueue("root.d1.d2.d3"); + + // Create 4 depth will fail. + try { + createQueue("root.e1.e2.e3.e4"); + Assert.fail("Can't exceed max queue depth."); + } catch (Exception ex) { + Assert.assertTrue(ex + instanceof SchedulerDynamicEditException); + } + + // Set max depth to 1 for root.d1 + csConf.setAutoCreatedQueuesV2MaxDepthLimit("root.d1", 1); + cs.reinitialize(csConf, mockRM.getRMContext()); + // Create 2 depth for root.d1 will fail. + try { + createQueue("root.d1.d_1.d_2"); + Assert.fail("Can't exceed max queue depth."); + } catch (Exception ex) { + Assert.assertTrue(ex + instanceof SchedulerDynamicEditException); + } + } + private LeafQueue createQueue(String queuePath) throws YarnException { return autoQueueHandler.autoCreateQueue( CSQueueUtils.extractQueuePath(queuePath));