diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index ffe862fc618..b1bef245b79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -190,22 +190,7 @@ protected void setupQueueConfigs(Resource clusterResource, usersManager.setUserLimit(conf.getUserLimit(getQueuePath())); usersManager.setUserLimitFactor(conf.getUserLimitFactor(getQueuePath())); - maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath()); - if (maxApplications < 0) { - int maxGlobalPerQueueApps = schedConf - .getGlobalMaximumApplicationsPerQueue(); - if (maxGlobalPerQueueApps > 0) { - maxApplications = maxGlobalPerQueueApps; - } else { - int maxSystemApps = schedConf. - getMaximumSystemApplications(); - maxApplications = - (int) (maxSystemApps * queueCapacities.getAbsoluteCapacity()); - } - } - maxApplicationsPerUser = Math.min(maxApplications, - (int) (maxApplications * (usersManager.getUserLimit() / 100.0f) - * usersManager.getUserLimitFactor())); + calculateMaxApplications(conf, schedConf); maxAMResourcePerQueuePercent = conf.getMaximumApplicationMasterResourcePerQueuePercent( @@ -336,6 +321,26 @@ protected void setupQueueConfigs(Resource clusterResource, } } + public void calculateMaxApplications(CapacitySchedulerConfiguration conf, + CapacitySchedulerConfiguration schedConf) { + maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath()); + if (maxApplications < 0) { + int maxGlobalPerQueueApps = schedConf + .getGlobalMaximumApplicationsPerQueue(); + if (maxGlobalPerQueueApps > 0) { + maxApplications = maxGlobalPerQueueApps; + } else { + int maxSystemApps = schedConf. + getMaximumSystemApplications(); + maxApplications = + (int) (maxSystemApps * queueCapacities.getAbsoluteCapacity()); + } + } + maxApplicationsPerUser = Math.min(maxApplications, + (int) (maxApplications * (usersManager.getUserLimit() / 100.0f) + * usersManager.getUserLimitFactor())); + } + /** * Used only by tests. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index e32130fc4ed..73f8ce640ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -1110,18 +1110,10 @@ private void deriveCapacityFromAbsoluteConfigurations(String label, if (childQueue instanceof LeafQueue) { LeafQueue leafQueue = (LeafQueue) childQueue; CapacitySchedulerConfiguration conf = csContext.getConfiguration(); - int maxApplications = (int) (conf.getMaximumSystemApplications() - * childQueue.getQueueCapacities().getAbsoluteCapacity(label)); - leafQueue.setMaxApplications(maxApplications); - - int maxApplicationsPerUser = Math.min(maxApplications, - (int) (maxApplications - * (leafQueue.getUsersManager().getUserLimit() / 100.0f) - * leafQueue.getUsersManager().getUserLimitFactor())); - leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser); + leafQueue.calculateMaxApplications(conf, conf); LOG.info("LeafQueue:" + leafQueue.getQueueName() + ", maxApplications=" - + maxApplications + ", maxApplicationsPerUser=" - + maxApplicationsPerUser + ", Abs Cap:" + + leafQueue.getMaxApplications() + ", maxApplicationsPerUser=" + + leafQueue.getMaxApplicationsPerUser() + ", Abs Cap:" + childQueue.getQueueCapacities().getAbsoluteCapacity(label)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index 298e1abcbd0..47598f83e69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -627,4 +627,80 @@ public void testEffectiveResourceAfterIncreasingClusterResource() rm.stop(); } + + @Test + public void testMaxApplications() throws Exception { + // create conf with basic queue configuration. + CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration( + false); + setupMinMaxResourceConfiguration(csConf); + + csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + @SuppressWarnings("resource") + MockRM rm = new MockRM(csConf); + rm.start(); + + // Add few nodes + rm.registerNode("127.0.0.1:1234", 250 * GB, 40); + + // Get queue object to verify min/max resource configuration. + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA); + Assert.assertNotNull(qA); + LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB); + Assert.assertNotNull(qB); + LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); + Assert.assertNotNull(qC); + + /* + * maximum applications should be calculated as + * (queue-capacity * maximum-system-applications) without global maximum + * applications and specified maximum applications for queues + */ + Assert.assertEquals(4000, qA.getMaxApplications()); + Assert.assertEquals(4000, qA.getMaxApplicationsPerUser()); + Assert.assertEquals(2000, qB.getMaxApplications()); + Assert.assertEquals(2000, qB.getMaxApplicationsPerUser()); + Assert.assertEquals(2000, qC.getMaxApplications()); + Assert.assertEquals(2000, qC.getMaxApplicationsPerUser()); + + /* + * maximum applications should equal global maximum applications + * if configured + */ + csConf.setGlobalMaximumApplicationsPerQueue(100); + cs.reinitialize(csConf, rm.getRMContext()); + Assert.assertEquals(100, qA.getMaxApplications()); + Assert.assertEquals(100, qA.getMaxApplicationsPerUser()); + Assert.assertEquals(100, qB.getMaxApplications()); + Assert.assertEquals(100, qB.getMaxApplicationsPerUser()); + Assert.assertEquals(100, qC.getMaxApplications()); + Assert.assertEquals(100, qC.getMaxApplicationsPerUser()); + + /* + * Test maximum applications should equal specified maximum applications + * for queues if configured + */ + csConf.setInt(CapacitySchedulerConfiguration.PREFIX + + CapacitySchedulerConfiguration.ROOT + + CapacitySchedulerConfiguration.DOT + QUEUEA + + CapacitySchedulerConfiguration.DOT + + CapacitySchedulerConfiguration.MAXIMUM_APPLICATIONS_SUFFIX, 999); + csConf.setInt(CapacitySchedulerConfiguration.PREFIX + + CapacitySchedulerConfiguration.ROOT + + CapacitySchedulerConfiguration.DOT + QUEUEB + + CapacitySchedulerConfiguration.DOT + + CapacitySchedulerConfiguration.MAXIMUM_APPLICATIONS_SUFFIX, 888); + cs.reinitialize(csConf, rm.getRMContext()); + Assert.assertEquals(999, qA.getMaxApplications()); + Assert.assertEquals(999, qA.getMaxApplicationsPerUser()); + Assert.assertEquals(888, qB.getMaxApplications()); + Assert.assertEquals(888, qB.getMaxApplicationsPerUser()); + Assert.assertEquals(100, qC.getMaxApplications()); + Assert.assertEquals(100, qC.getMaxApplicationsPerUser()); + + rm.stop(); + } }