Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java (date 1536642842000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java (date 1536660408000) @@ -53,6 +53,8 @@ */ public class TestFairSchedulerConfiguration { + private static final int TEST_INCREMENT_MEMORY_SIZE = 7; + private static final int TEST_INCREMENT_VCORES = 8; private static final String A_CUSTOM_RESOURCE = "a-custom-resource"; private static class CustomResourceTypesConfigurationProvider @@ -467,7 +469,7 @@ testAppender.getLogEvents().stream().anyMatch( e -> e.getLevel() == Level.WARN && ("Configuration " + "yarn.resource-types.memory-mb.increment-allocation=13 is " + - "overriding the yarn.scheduler.increment-allocation-mb=7 " + + "overriding the yarn.scheduler.fair.increment-allocation-mb=7 " + "property").equals(e.getMessage()))); } finally { logger.getLogger().removeAppender(testAppender); @@ -493,10 +495,48 @@ testAppender.getLogEvents().stream().anyMatch( e -> e.getLevel() == Level.WARN && ("Configuration " + "yarn.resource-types.vcores.increment-allocation=13 is " + - "overriding the yarn.scheduler.increment-allocation-vcores=7 " + - "property").equals(e.getMessage()))); + "overriding the yarn.scheduler.fair.increment-allocation-vcores" + + "=7 property").equals(e.getMessage()))); } finally { logger.getLogger().removeAppender(testAppender); } } + + @Test + public void testDeprecatedResourceTypesConfigs() { + // SETUP + Configuration conf = new Configuration(); + conf.set("yarn.scheduler.increment-allocation-mb", + String.valueOf(TEST_INCREMENT_MEMORY_SIZE)); + conf.set("yarn.scheduler.increment-allocation-vcores", + String.valueOf(TEST_INCREMENT_VCORES)); + + // ACT + FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); + + // ASSERT + assertEquals(fsc.getIncrementAllocation().getMemorySize(), + TEST_INCREMENT_MEMORY_SIZE); + assertEquals(fsc.getIncrementAllocation().getVirtualCores(), + TEST_INCREMENT_VCORES); + } + + @Test + public void testNewResourceTypesConfigs() { + // SETUP + Configuration conf = new Configuration(); + conf.set("yarn.scheduler.fair.increment-allocation-mb", + String.valueOf(TEST_INCREMENT_MEMORY_SIZE)); + conf.set("yarn.scheduler.fair.increment-allocation-vcores", + String.valueOf(TEST_INCREMENT_VCORES)); + + // ACT + FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); + + // ASSERT + assertEquals(fsc.getIncrementAllocation().getMemorySize(), + TEST_INCREMENT_MEMORY_SIZE); + assertEquals(fsc.getIncrementAllocation().getVirtualCores(), + TEST_INCREMENT_VCORES); + } } Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md (date 1536642842000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md (date 1536659609000) @@ -73,12 +73,12 @@ | `yarn.scheduler.fair.locality.threshold.node` | For applications that request containers on particular nodes, the number of scheduling opportunities since the last container assignment to wait before accepting a placement on another node. Expressed as a float between 0 and 1, which, as a fraction of the cluster size, is the number of scheduling opportunities to pass up. The default value of -1.0 means don't pass up any scheduling opportunities. | | `yarn.scheduler.fair.locality.threshold.rack` | For applications that request containers on particular racks, the number of scheduling opportunities since the last container assignment to wait before accepting a placement on another rack. Expressed as a float between 0 and 1, which, as a fraction of the cluster size, is the number of scheduling opportunities to pass up. The default value of -1.0 means don't pass up any scheduling opportunities. | | `yarn.scheduler.fair.allow-undeclared-pools` | If this is true, new queues can be created at application submission time, whether because they are specified as the application's queue by the submitter or because they are placed there by the user-as-default-queue property. If this is false, any time an app would be placed in a queue that is not specified in the allocations file, it is placed in the "default" queue instead. Defaults to true. If a queue placement policy is given in the allocations file, this property is ignored. | +| `yarn.scheduler.fair.increment-allocation-mb` | The allocation increment for memory. Deprecates `yarn.scheduler.increment-allocation-mb`. No longer preferred. Use `yarn.resource-types.memory-mb.increment-allocation` instead. Defaults to 1024 MB. | +| `yarn.scheduler.fair.increment-allocation-vcores` | The allocation increment for CPU vcores. Deprecates `yarn.scheduler.increment-allocation-vcores`. No longer preferred. Use `yarn.resource-types.vcores.increment-allocation` instead. Defaults to 1. | | `yarn.scheduler.fair.update-interval-ms` | The interval at which to lock the scheduler and recalculate fair shares, recalculate demand, and check whether anything is due for preemption. Defaults to 500 ms. | | `yarn.resource-types.memory-mb.increment-allocation` | The fairscheduler grants memory in increments of this value. If you submit a task with resource request that is not a multiple of `memory-mb.increment-allocation`, the request will be rounded up to the nearest increment. Defaults to 1024 MB. | | `yarn.resource-types.vcores.increment-allocation` | The fairscheduler grants vcores in increments of this value. If you submit a task with resource request that is not a multiple of `vcores.increment-allocation`, the request will be rounded up to the nearest increment. Defaults to 1. | | `yarn.resource-types..increment-allocation` | The fairscheduler grants `` in increments of this value. If you submit a task with resource request that is not a multiple of `.increment-allocation`, the request will be rounded up to the nearest increment. If this property is not specified for a resource, the increment round-up will not be applied. If no unit is specified, the default unit for the resource is assumed. | -| `yarn.scheduler.increment-allocation-mb` | The allocation increment for memory. No longer preferred. Use `yarn.resource-types.memory-mb.increment-allocation` instead. Defaults to 1024 MB. | -| `yarn.scheduler.increment-allocation-vcores` | The allocation increment for CPU vcores. No longer preferred. Use `yarn.resource-types.vcores.increment-allocation` instead. Defaults to 1. | ###Allocation file format Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java (date 1536642842000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java (date 1536660821000) @@ -55,7 +55,7 @@ */ @Deprecated public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_MB = - YarnConfiguration.YARN_PREFIX + "scheduler.increment-allocation-mb"; + YarnConfiguration.YARN_PREFIX + "scheduler.fair.increment-allocation-mb"; @Deprecated public static final int DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB = 1024; /** @@ -67,7 +67,8 @@ */ @Deprecated public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES = - YarnConfiguration.YARN_PREFIX + "scheduler.increment-allocation-vcores"; + YarnConfiguration.YARN_PREFIX + + "scheduler.fair.increment-allocation-vcores"; @Deprecated public static final int DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES = 1; @@ -217,6 +218,22 @@ private static final String INVALID_RESOURCE_DEFINITION_PREFIX = "Error reading resource config--invalid resource definition: "; + static { + addDeprecatedKeys(); + } + + private static void addDeprecatedKeys() { + Configuration.addDeprecations( + new DeprecationDelta[]{ + new DeprecationDelta(YarnConfiguration.YARN_PREFIX + + "scheduler.increment-allocation-mb", + RM_SCHEDULER_INCREMENT_ALLOCATION_MB), + new DeprecationDelta(YarnConfiguration.YARN_PREFIX + + "scheduler.increment-allocation-vcores", + RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES), + }); + } + public FairSchedulerConfiguration() { super(); }