diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index be9a0e36391..a0c0ee38f17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -362,7 +362,8 @@ protected void setupQueueConfigs(Resource clusterResource, try { if (isDynamicQueue() && getParent() instanceof ParentQueue) { ((ParentQueue) getParent()).getAutoCreatedQueueTemplate() - .setTemplateEntriesForChild(configuration, getQueuePath()); + .setTemplateEntriesForChild(getQueuePath(), configuration, + csContext.getConfiguration()); } // get labels this.accessibleLabels = @@ -1703,5 +1704,4 @@ public void setLastSubmittedTimestamp(long lastSubmittedTimestamp) { writeLock.unlock(); } } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java index 203ec4dcd6f..bdc8626e3fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java @@ -64,17 +64,20 @@ public static String getAutoQueueTemplatePrefix(String queue) { /** * Sets the configuration properties of a child queue based on its parent * template settings. - * @param conf configuration to set + * @param newConfiguration newly parsed configuration + * @param oldConfiguration previously parsed configuration generally set + * in {@code CapacitySchedulerContext} * @param childQueuePath child queue path used for prefixing the properties */ - public void setTemplateEntriesForChild(Configuration conf, - String childQueuePath) { + public void setTemplateEntriesForChild(String childQueuePath, + Configuration newConfiguration, + Configuration oldConfiguration) { if (childQueuePath.equals(ROOT)) { return; } // Get all properties that are explicitly set - Set alreadySetProps = conf.getPropsWithPrefix( + Set alreadySetProps = newConfiguration.getPropsWithPrefix( CapacitySchedulerConfiguration.getQueuePrefix(childQueuePath)).keySet(); for (Map.Entry entry : templateProperties.entrySet()) { @@ -82,7 +85,11 @@ public void setTemplateEntriesForChild(Configuration conf, if (alreadySetProps.contains(entry.getKey())) { continue; } - conf.set(CapacitySchedulerConfiguration.getQueuePrefix( + newConfiguration.set(CapacitySchedulerConfiguration.getQueuePrefix( + childQueuePath) + entry.getKey(), entry.getValue()); + // Since templates are set at runtime we need to set these properties + // in previously parsed config + oldConfiguration.set(CapacitySchedulerConfiguration.getQueuePrefix( childQueuePath) + entry.getKey(), entry.getValue()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 6b6dd5afd3e..a70d332ffc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -482,8 +482,8 @@ private CapacitySchedulerConfiguration getConfForAutoCreatedQueue( CapacitySchedulerConfiguration dupCSConfig = new CapacitySchedulerConfiguration( csContext.getConfiguration(), false); - autoCreatedQueueTemplate.setTemplateEntriesForChild(dupCSConfig, - childQueuePath); + autoCreatedQueueTemplate.setTemplateEntriesForChild(childQueuePath, + dupCSConfig, csContext.getConfiguration()); if (isLeaf) { // set to -1, to disable it dupCSConfig.setUserLimitFactor(childQueuePath, -1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java index 2763af026a6..8a6c1e51c2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java @@ -43,7 +43,7 @@ public void testNonWildCardTemplate() { conf.set(getTemplateKey(TEST_QUEUE_AB, "capacity"), "6w"); AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); + template.setTemplateEntriesForChild(TEST_QUEUE_ABC, conf, conf); Assert.assertEquals("weight is not set", 6f, conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); @@ -55,7 +55,7 @@ public void testOneLevelWildcardTemplate() { conf.set(getTemplateKey("root.a.*", "capacity"), "6w"); AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); + template.setTemplateEntriesForChild(TEST_QUEUE_ABC, conf, conf); Assert.assertEquals("weight is not set", 6f, conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); @@ -67,7 +67,7 @@ public void testIgnoredWhenRootWildcarded() { conf.set(getTemplateKey("*", "capacity"), "6w"); AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, ROOT); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_A); + template.setTemplateEntriesForChild(TEST_QUEUE_A, conf, conf); Assert.assertEquals("weight is set", -1f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); @@ -78,7 +78,7 @@ public void testIgnoredWhenNoParent() { conf.set(getTemplateKey("root", "capacity"), "6w"); AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, ROOT); - template.setTemplateEntriesForChild(conf, ROOT); + template.setTemplateEntriesForChild(ROOT, conf, conf); Assert.assertEquals("weight is set", -1f, conf.getNonLabeledQueueWeight(ROOT), 10e-6); @@ -92,7 +92,7 @@ public void testTemplatePrecedence() { AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); + template.setTemplateEntriesForChild(TEST_QUEUE_ABC, conf, conf); Assert.assertEquals( "explicit template does not have the highest precedence", 6f, @@ -103,7 +103,7 @@ public void testTemplatePrecedence() { newConf.set(getTemplateKey("root.a.*", "capacity"), "4w"); template = new AutoCreatedQueueTemplate(newConf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(newConf, TEST_QUEUE_ABC); + template.setTemplateEntriesForChild(TEST_QUEUE_ABC, newConf, conf); Assert.assertEquals("precedence is invalid", 4f, newConf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); @@ -115,7 +115,7 @@ public void testRootTemplate() { AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, ROOT); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_A); + template.setTemplateEntriesForChild(TEST_QUEUE_A, conf, conf); Assert.assertEquals("root property is not set", 2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index b96c1e48968..e1d035e8b0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -696,6 +696,13 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception { cs.reinitialize(csConf, mockRM.getRMContext()); Assert.assertEquals("weight is not explicitly set", 4f, a2.getQueueCapacities().getWeight(), 1e-6); + + csConf.setBoolean(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix( + "root.a") + CapacitySchedulerConfiguration.AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE, false); + cs.reinitialize(csConf, mockRM.getRMContext()); + LeafQueue a3 = createQueue("root.a.a3"); + Assert.assertFalse("auto queue deletion should be turned off on a3", + a3.isEligibleForAutoDeletion()); } @Test