diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index a7602343db2..1c26480e1d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -126,7 +126,7 @@ public void convert(FSConfigToCSConfigConverterParams params) Configuration inputYarnSiteConfig = getInputYarnSiteConfig(params); handleFairSchedulerConfig(params, inputYarnSiteConfig); - convert(inputYarnSiteConfig); + convert(inputYarnSiteConfig, usePercentages); } private void prepareOutputFiles(boolean console) @@ -222,7 +222,8 @@ private void handleFairSchedulerConfig( } @VisibleForTesting - void convert(Configuration inputYarnSiteConfig) throws Exception { + void convert(Configuration inputYarnSiteConfig, + boolean usePercentages) throws Exception { // initialize Fair Scheduler RMContext ctx = new RMContextImpl(); PlacementManager placementManager = new PlacementManager(); @@ -253,7 +254,7 @@ void convert(Configuration inputYarnSiteConfig) throws Exception { capacitySchedulerConfig = new CapacitySchedulerConfiguration(new Configuration(false)); - convertYarnSiteXml(inputYarnSiteConfig); + convertYarnSiteXml(inputYarnSiteConfig, usePercentages); convertCapacitySchedulerXml(fs); if (convertPlacementRules) { @@ -272,12 +273,13 @@ void convert(Configuration inputYarnSiteConfig) throws Exception { convertedYarnSiteConfig.writeXml(yarnSiteOutputStream); } - private void convertYarnSiteXml(Configuration inputYarnSiteConfig) { + private void convertYarnSiteXml(Configuration inputYarnSiteConfig, + boolean usePercentages) { FSYarnSiteConverter siteConverter = new FSYarnSiteConverter(); siteConverter.convertSiteProperties(inputYarnSiteConfig, convertedYarnSiteConfig, drfUsed, - conversionOptions.isEnableAsyncScheduler()); + conversionOptions.isEnableAsyncScheduler(), usePercentages); preemptionEnabled = siteConverter.isPreemptionEnabled(); sizeBasedWeight = siteConverter.isSizeBasedWeight(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java index 0e9b3894885..4dab761e073 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -78,7 +78,12 @@ public void convertQueueHierarchy(FSQueue queue) { emitMaxAMShare(queueName, queue); emitMaxParallelApps(queueName, queue); emitMaxAllocations(queueName, queue); - emitPreemptionDisabled(queueName, queue); + + if (!preemptionEnabled) { + disabledAllPreemption(); + } else { + emitPreemptionDisabled(queueName, queue); + } emitChildCapacity(queue); emitMaximumCapacity(queueName, queue); @@ -215,6 +220,16 @@ private void emitPreemptionDisabled(String queueName, FSQueue queue) { } } + /** + * Only used for that + * when scheduler monitor enabled, we should use + * this to disable preemption in cs. + */ + private void disabledAllPreemption() { + capacitySchedulerConfig.set(PREFIX + "root.disable_preemption", + "true"); + } + /** * yarn.scheduler.fair.sizebasedweight ==> * yarn.scheduler.capacity.<queue-path> diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java index 4222e3af6c2..964a19e3a6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueDeletionPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; @@ -36,7 +37,8 @@ @SuppressWarnings({"deprecation", "checkstyle:linelength"}) public void convertSiteProperties(Configuration conf, - Configuration yarnSiteConfig, boolean drfUsed, boolean enableAsyncScheduler) { + Configuration yarnSiteConfig, boolean drfUsed, + boolean enableAsyncScheduler, boolean userPercentage) { yarnSiteConfig.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getCanonicalName()); @@ -52,10 +54,32 @@ public void convertSiteProperties(Configuration conf, "schedule-asynchronously.scheduling-interval-ms", interval); } + // This should be always true to trigger cs auto + // refresh queue. + yarnSiteConfig.setBoolean( + YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); + + if (!userPercentage) { + String policies = + yarnSiteConfig.get(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES); + if (policies == null) { + policies = AutoCreatedQueueDeletionPolicy. + class.getCanonicalName(); + } else { + policies += "," + AutoCreatedQueueDeletionPolicy. + class.getCanonicalName(); + } + + yarnSiteConfig.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, + policies); + + // Set the expired for deletion interval to 10s, consistent with fs. + yarnSiteConfig.setInt(CapacitySchedulerConfiguration. + AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME, 10); + } + if (conf.getBoolean(FairSchedulerConfiguration.PREEMPTION, FairSchedulerConfiguration.DEFAULT_PREEMPTION)) { - yarnSiteConfig.setBoolean( - YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); preemptionEnabled = true; int waitTimeBeforeKill = conf.getInt( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 5450d409bda..a75f0e9830a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -169,7 +169,7 @@ private void createConverter() { @Test public void testDefaultMaxAMShare() throws Exception { - converter.convert(config); + converter.convert(config, false); Configuration conf = converter.getCapacitySchedulerConfig(); String maxAmShare = @@ -216,7 +216,7 @@ public void testDefaultMaxAMShareDisabled() throws Exception { @Test public void testConvertACLs() throws Exception { - converter.convert(config); + converter.convert(config, false); Configuration conf = converter.getCapacitySchedulerConfig(); @@ -253,7 +253,7 @@ public void testConvertACLs() throws Exception { @Test public void testDefaultQueueMaxParallelApps() throws Exception { - converter.convert(config); + converter.convert(config, false); Configuration conf = converter.getCapacitySchedulerConfig(); @@ -263,7 +263,7 @@ public void testDefaultQueueMaxParallelApps() throws Exception { @Test public void testSpecificQueueMaxParallelApps() throws Exception { - converter.convert(config); + converter.convert(config, false); Configuration conf = converter.getCapacitySchedulerConfig(); @@ -273,7 +273,7 @@ public void testSpecificQueueMaxParallelApps() throws Exception { @Test public void testDefaultUserMaxParallelApps() throws Exception { - converter.convert(config); + converter.convert(config, false); Configuration conf = converter.getCapacitySchedulerConfig(); int userMaxParallelApps = @@ -286,7 +286,7 @@ public void testDefaultUserMaxParallelApps() throws Exception { @Test public void testSpecificUserMaxParallelApps() throws Exception { - converter.convert(config); + converter.convert(config, false); Configuration conf = converter.getCapacitySchedulerConfig(); @@ -308,7 +308,7 @@ public void testQueueMaxChildCapacityNotSupported() throws Exception { Mockito.doThrow(new UnsupportedPropertyException("test")) .when(ruleHandler).handleMaxChildCapacity(); - converter.convert(config); + converter.convert(config, false); } @Test @@ -320,7 +320,7 @@ public void testReservationSystemNotSupported() throws Exception { .when(ruleHandler).handleMaxChildCapacity(); config.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); - converter.convert(config); + converter.convert(config, false); } @Test @@ -535,7 +535,7 @@ public void testConversionWithInvalidPlacementRules() throws Exception { config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true); expectedException.expect(ServiceStateException.class); - converter.convert(config); + converter.convert(config, false); } @Test @@ -619,7 +619,7 @@ private void testUserAsDefaultQueueAndPlacementRules( converter.setConvertPlacementRules(true); converter.setConsoleMode(true); - converter.convert(config); + converter.convert(config, false); String json = converter.getCapacitySchedulerConfig() .get(CapacitySchedulerConfiguration.MAPPING_RULE_JSON); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java index e648c1c91cd..d3d2ac14c9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java @@ -512,6 +512,10 @@ public void testReservationSystemNotSupported() { private void assertNoValueForQueues(Set queues, String postfix, Configuration config) { for (String queue : queues) { + // Skip for root. + if (queue.equals("root")) { + continue; + } String key = PREFIX + queue + postfix; assertNull("Key " + key + " has value, but it should be null", config.get(key)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java index 9cebf16f8be..ff1f616fb9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java @@ -18,6 +18,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueDeletionPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; @@ -25,9 +26,7 @@ import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; /** * Unit tests for FSYarnSiteConverter. @@ -54,7 +53,7 @@ public void testSiteContinuousSchedulingConversion() { FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_SLEEP_MS, 666); converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, - false); + false, false); assertTrue("Cont. scheduling", yarnConvertedConfig.getBoolean( CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false)); @@ -73,7 +72,7 @@ public void testSitePreemptionConversion() { 321); converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, - false); + false, false); assertTrue("Preemption enabled", yarnConvertedConfig.getBoolean( @@ -94,7 +93,7 @@ public void testSiteAssignMultipleConversion() { yarnConfig.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, - false); + false, false); assertTrue("Assign multiple", yarnConvertedConfig.getBoolean( @@ -107,7 +106,7 @@ public void testSiteMaxAssignConversion() { yarnConfig.setInt(FairSchedulerConfiguration.MAX_ASSIGN, 111); converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, - false); + false, false); assertEquals("Max assign", 111, yarnConvertedConfig.getInt( @@ -122,7 +121,7 @@ public void testSiteLocalityThresholdConversion() { "321.321"); converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, - false); + false, false); assertEquals("Locality threshold node", "123.123", yarnConvertedConfig.get( @@ -135,7 +134,7 @@ public void testSiteLocalityThresholdConversion() { @Test public void testSiteDrfEnabledConversion() { converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, true, - false); + false, false); assertEquals("Resource calculator type", DominantResourceCalculator.class, yarnConvertedConfig.getClass( @@ -145,7 +144,7 @@ public void testSiteDrfEnabledConversion() { @Test public void testSiteDrfDisabledConversion() { converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, - false); + false, false); assertEquals("Resource calculator type", DefaultResourceCalculator.class, yarnConvertedConfig.getClass( @@ -156,7 +155,7 @@ public void testSiteDrfDisabledConversion() { @Test public void testAsyncSchedulingEnabledConversion() { converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, true, - true); + true, false); assertTrue("Asynchronous scheduling", yarnConvertedConfig.getBoolean( CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, @@ -166,10 +165,83 @@ public void testAsyncSchedulingEnabledConversion() { @Test public void testAsyncSchedulingDisabledConversion() { converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, - false); + false, false); assertFalse("Asynchronous scheduling", yarnConvertedConfig.getBoolean( CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, CapacitySchedulerConfiguration.DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE)); } + + @Test + public void testSiteQueueAutoDeletionConversionWithWeightMode() { + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, + false, false); + assertTrue(yarnConvertedConfig.get(YarnConfiguration. + RM_SCHEDULER_ENABLE_MONITORS), true); + assertTrue("Scheduling Policies contain auto deletion policy", + yarnConvertedConfig. + get(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES) + .contains(AutoCreatedQueueDeletionPolicy. + class.getCanonicalName())); + + // Test when policy has existed. + yarnConvertedConfig. + set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, + "testPolicy"); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, + false, false); + assertTrue("Scheduling Policies contain auto deletion policy", + yarnConvertedConfig. + get(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES) + .contains(AutoCreatedQueueDeletionPolicy. + class.getCanonicalName())); + + assertEquals("Auto deletion policy expired time should be 10s", + 10, yarnConvertedConfig. + getLong(CapacitySchedulerConfiguration. + AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME, + CapacitySchedulerConfiguration. + DEFAULT_AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME)); + } + + @Test + public void + testSiteQueueAutoDeletionConversionDisabledForPercentageMode() { + + // test percentage mode + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, + false, true); + assertTrue(yarnConvertedConfig.get(YarnConfiguration. + RM_SCHEDULER_ENABLE_MONITORS), true); + + assertTrue("Scheduling Policies should not" + + "contain auto deletion policy in percentage mode", + yarnConvertedConfig. + get(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES) == null || + !yarnConvertedConfig. + get(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES). + contains(AutoCreatedQueueDeletionPolicy. + class.getCanonicalName())); + + yarnConvertedConfig. + set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, + "testPolicy"); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false, + false, true); + assertFalse("Scheduling Policies should not " + + "contain auto deletion policy in percentage mode", + yarnConvertedConfig. + get(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES) + .contains(AutoCreatedQueueDeletionPolicy. + class.getCanonicalName())); + + assertNotEquals("Auto deletion policy expired time should not " + + "be set in percentage mode", + 10, yarnConvertedConfig. + getLong(CapacitySchedulerConfiguration. + AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME, + CapacitySchedulerConfiguration. + DEFAULT_AUTO_CREATE_CHILD_QUEUE_EXPIRED_TIME)); + + } }