diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index be9a0e36391..a48caa301c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -360,10 +360,7 @@ protected void setupQueueConfigs(Resource clusterResource, writeLock.lock(); try { - if (isDynamicQueue() && getParent() instanceof ParentQueue) { - ((ParentQueue) getParent()).getAutoCreatedQueueTemplate() - .setTemplateEntriesForChild(configuration, getQueuePath()); - } + setDynamicQueueProperties(configuration); // get labels this.accessibleLabels = configuration.getAccessibleNodeLabels(getQueuePath()); @@ -478,6 +475,19 @@ protected void setupQueueConfigs(Resource clusterResource, } } + /** + * Set properties specific to dynamic queues. + * @param configuration configuration on which the properties are set + */ + protected void setDynamicQueueProperties( + CapacitySchedulerConfiguration configuration) { + // Set properties from parent template + if (isDynamicQueue() && getParent() instanceof ParentQueue) { + ((ParentQueue) getParent()).getAutoCreatedQueueTemplate() + .setTemplateEntriesForChild(configuration, getQueuePath()); + } + } + private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) { String myQueuePath = getQueuePath(); Resource clusterMax = ResourceUtils diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 71732f65e34..488bf5b38c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -147,14 +147,21 @@ @SuppressWarnings({ "unchecked", "rawtypes" }) public LeafQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { - this(cs, cs.getConfiguration(), queueName, parent, old); + this(cs, cs.getConfiguration(), queueName, parent, old, false); } public LeafQueue(CapacitySchedulerContext cs, CapacitySchedulerConfiguration configuration, - String queueName, CSQueue parent, CSQueue old) throws + String queueName, CSQueue parent, CSQueue old) throws IOException { + this(cs, configuration, queueName, parent, old, false); + } + + public LeafQueue(CapacitySchedulerContext cs, + CapacitySchedulerConfiguration configuration, + String queueName, CSQueue parent, CSQueue old, boolean isDynamic) throws IOException { super(cs, configuration, queueName, parent, old); + setDynamicQueue(isDynamic); this.scheduler = cs; this.usersManager = new UsersManager(metrics, this, labelManager, scheduler, @@ -1691,6 +1698,19 @@ protected boolean canAssignToUser(Resource clusterResource, } } + @Override + protected void setDynamicQueueProperties( + CapacitySchedulerConfiguration configuration) { + super.setDynamicQueueProperties(configuration); + if (isDynamicQueue()) { + // set to -1, to disable it + configuration.setUserLimitFactor(getQueuePath(), -1); + // Set Max AM percentage to a higher value + configuration.setMaximumApplicationMasterResourcePerQueuePercent( + getQueuePath(), 0.5f); + } + } + private void updateSchedulerHealthForCompletedContainer( RMContainer rmContainer, ContainerStatus containerStatus) { // Update SchedulerHealth for released / preempted container diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 6b6dd5afd3e..e661c899536 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -112,14 +112,21 @@ public ParentQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { - this(cs, cs.getConfiguration(), queueName, parent, old); + this(cs, cs.getConfiguration(), queueName, parent, old, false); + } + private ParentQueue(CapacitySchedulerContext cs, + CapacitySchedulerConfiguration csConf, String queueName, + CSQueue parent, + CSQueue old) throws IOException { + this(cs, csConf, queueName, parent, old, false); } private ParentQueue(CapacitySchedulerContext cs, CapacitySchedulerConfiguration csConf, String queueName, CSQueue parent, - CSQueue old) + CSQueue old, boolean isDynamic) throws IOException { super(cs, queueName, parent, old); + setDynamicQueue(isDynamic); this.scheduler = cs; this.rootQueue = (parent == null); @@ -476,26 +483,6 @@ public String toString() { "numContainers=" + getNumContainers(); } - private CapacitySchedulerConfiguration getConfForAutoCreatedQueue( - String childQueuePath, boolean isLeaf) { - // Copy existing config - CapacitySchedulerConfiguration dupCSConfig = - new CapacitySchedulerConfiguration( - csContext.getConfiguration(), false); - autoCreatedQueueTemplate.setTemplateEntriesForChild(dupCSConfig, - childQueuePath); - if (isLeaf) { - // set to -1, to disable it - dupCSConfig.setUserLimitFactor(childQueuePath, -1); - - // Set Max AM percentage to a higher value - dupCSConfig.setMaximumApplicationMasterResourcePerQueuePercent( - childQueuePath, 0.5f); - } - - return dupCSConfig; - } - private CSQueue createNewQueue(String childQueuePath, boolean isLeaf) throws SchedulerDynamicEditException { try { @@ -504,13 +491,11 @@ private CSQueue createNewQueue(String childQueuePath, boolean isLeaf) childQueuePath.lastIndexOf(".") + 1); if (isLeaf) { - childQueue = new LeafQueue(csContext, - getConfForAutoCreatedQueue(childQueuePath, isLeaf), queueShortName, - this, null); + childQueue = new LeafQueue(csContext, csContext.getConfiguration(), + queueShortName, this, null, true); } else{ - childQueue = new ParentQueue(csContext, - getConfForAutoCreatedQueue(childQueuePath, isLeaf), queueShortName, - this, null); + childQueue = new ParentQueue(csContext, csContext.getConfiguration(), + queueShortName, this, null, true); } childQueue.setDynamicQueue(true); // It should be sufficient now, we don't need to set more, because weights diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index b96c1e48968..5539d4aa8a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -696,6 +696,14 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception { cs.reinitialize(csConf, mockRM.getRMContext()); Assert.assertEquals("weight is not explicitly set", 4f, a2.getQueueCapacities().getWeight(), 1e-6); + + csConf.setBoolean(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix( + "root.a") + CapacitySchedulerConfiguration + .AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE, false); + cs.reinitialize(csConf, mockRM.getRMContext()); + LeafQueue a3 = createQueue("root.a.a3"); + Assert.assertFalse("auto queue deletion should be turned off on a3", + a3.isEligibleForAutoDeletion()); } @Test