diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java index 4222e3af6c2..86564d6d326 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueDeletionPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; @@ -52,10 +53,30 @@ public void convertSiteProperties(Configuration conf, "schedule-asynchronously.scheduling-interval-ms", interval); } + // This should be always true to trigger dynamic queue auto deletion + // when expired. + yarnSiteConfig.setBoolean( + YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); + + String policies = + yarnSiteConfig.get(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES); + if (policies == null) { + policies = AutoCreatedQueueDeletionPolicy. + class.getCanonicalName(); + } else { + policies += "," + AutoCreatedQueueDeletionPolicy. + class.getCanonicalName(); + } + + yarnSiteConfig.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, + policies); + + // Set the expired for deletion interval to 10s, consistent with fs. + yarnSiteConfig.setInt(CapacitySchedulerConfiguration. + AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME, 10); + if (conf.getBoolean(FairSchedulerConfiguration.PREEMPTION, FairSchedulerConfiguration.DEFAULT_PREEMPTION)) { - yarnSiteConfig.setBoolean( - YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); preemptionEnabled = true; int waitTimeBeforeKill = conf.getInt( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java index 9cebf16f8be..23dc2448c1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java @@ -18,6 +18,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueDeletionPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; @@ -172,4 +173,36 @@ public void testAsyncSchedulingDisabledConversion() { CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, CapacitySchedulerConfiguration.DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE)); } + + @Test + public void testSiteQueueAutoDeletionConversion() { + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, + false); + assertTrue(yarnConvertedConfig.get(YarnConfiguration. + RM_SCHEDULER_ENABLE_MONITORS), true); + assertTrue("Scheduling Policies contain auto deletion policy", + yarnConvertedConfig. + get(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES) + .contains(AutoCreatedQueueDeletionPolicy. + class.getCanonicalName())); + + // Test when policy has existed. + yarnConvertedConfig. + set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, + "testPolicy"); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, + false); + assertTrue("Scheduling Policies contain auto deletion policy", + yarnConvertedConfig. + get(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES) + .contains(AutoCreatedQueueDeletionPolicy. + class.getCanonicalName())); + + assertEquals("Auto deletion policy expired time should be 10s", + 10, yarnConvertedConfig. + getLong(CapacitySchedulerConfiguration. + AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME, + CapacitySchedulerConfiguration. + DEFAULT_AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME)); + } }