diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index eb9c283..6732ef0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -107,6 +107,14 @@ public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" + DEFAULT_RM_SCHEDULER_PORT; + /** Indicates if an allocation can have zero in some of their + capabilities. Even if set to true, at least one of the capabilities must + be greater than zero. */ + public static final String RM_SCHEDULER_ALLOW_ZERO_ALLOCATION = + YARN_PREFIX + "scheduler.allow-zero-allocation"; + public static final boolean DEFAULT_RM_SCHEDULER_ALLOW_ZERO_ALLOCATION = + false; + /** Increment request grant-able by the RM scheduler. */ public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_MB = YARN_PREFIX + "scheduler.increment-allocation-mb"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 55f9af3..6693d32 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -200,6 +200,14 @@ + Indicates if an allocation can have zero in some of their + capabilities. Even if set to true, at least one of the capabilities must + be greater than zero. + yarn.scheduler.allow-zero-allocation + false + + + The increment allocation for every container request at the RM, in MBs. Memory requests lower than this won't take effect, and the specified value will get allocated at increment. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 86f96f6..36cac42 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -301,7 +301,14 @@ public void setQueues(String queue, String[] subQueues) { } public Resource getMinimumAllocation() { - return getIncrementAllocation(); + Resource minimum; + if (getBoolean(YarnConfiguration.RM_SCHEDULER_ALLOW_ZERO_ALLOCATION, + YarnConfiguration.DEFAULT_RM_SCHEDULER_ALLOW_ZERO_ALLOCATION)) { + minimum = getIncrementAllocation(); + } else { + minimum = Resources.createResource(0, 0); + } + return minimum; } public Resource getMaximumAllocation() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 62be05e..87dbbf8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -83,7 +83,14 @@ public FairSchedulerConfiguration(Configuration conf) { } public Resource getMinimumAllocation() { - return getIncrementAllocation(); + Resource minimum; + if (getBoolean(YarnConfiguration.RM_SCHEDULER_ALLOW_ZERO_ALLOCATION, + YarnConfiguration.DEFAULT_RM_SCHEDULER_ALLOW_ZERO_ALLOCATION)) { + minimum = getIncrementAllocation(); + } else { + minimum = Resources.createResource(0, 0); + } + return minimum; } public Resource getMaximumAllocation() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 29fae23..c4188bb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -374,6 +374,36 @@ public void testParseQueue() throws IOException { } @Test + public void testDontAllowZeroResourcesSettings() throws IOException { + CapacityScheduler cs = new CapacityScheduler(); + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2); + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); + Assert.assertEquals(cs.getIncrementResourceCapability().getMemory(), + cs.getMinimumResourceCapability().getMemory()); + Assert.assertEquals(cs.getIncrementResourceCapability().getVirtualCores(), + cs.getMinimumResourceCapability().getVirtualCores()); + } + + @Test + public void testAllowZeroResourcesSettings() throws IOException { + CapacityScheduler cs = new CapacityScheduler(); + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ALLOW_ZERO_ALLOCATION, + YarnConfiguration.DEFAULT_RM_SCHEDULER_ALLOW_ZERO_ALLOCATION); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2); + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); + Assert.assertEquals(0, cs.getMinimumResourceCapability().getMemory()); + Assert.assertEquals(0, cs.getMinimumResourceCapability().getVirtualCores()); + } + + @Test public void testReconnectedNode() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index bc24645..f25232e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -63,4 +63,34 @@ public void testGibberish() throws Exception { parseResourceConfigValue("1o24vc0res"); } + @Test + public void testDontAllowZeroResourcesSettings() throws IOException { + FairScheduler fs = new FairScheduler(); + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2); + fs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); + Assert.assertEquals(fs.getIncrementResourceCapability().getMemory(), + fs.getMinimumResourceCapability().getMemory()); + Assert.assertEquals(fs.getIncrementResourceCapability().getVirtualCores(), + fs.getMinimumResourceCapability().getVirtualCores()); + } + + @Test + public void testAllowZeroResourcesSettings() throws IOException { + FairScheduler fs = new FairScheduler(); + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ALLOW_ZERO_ALLOCATION, + YarnConfiguration.DEFAULT_RM_SCHEDULER_ALLOW_ZERO_ALLOCATION); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2); + fs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); + Assert.assertEquals(0, fs.getMinimumResourceCapability().getMemory()); + Assert.assertEquals(0, fs.getMinimumResourceCapability().getVirtualCores()); + } + }