diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 2f6ca5a1597..b05fb504624 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -1528,21 +1528,30 @@ private Resource getMinResourceNormalized(String name, } void updateMaxAppRelatedField(CapacitySchedulerConfiguration conf, - LeafQueue leafQueue, String label) { + LeafQueue leafQueue) { int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath); + int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue(); + String maxLabel = RMNodeLabelsManager.NO_LABEL; + if (maxApplications < 0) { - int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue(); - if (maxGlobalPerQueueApps > 0) { - // In absolute mode, should - // shrink when change to corresponding label capacity. - maxApplications = this.capacityConfigType - != CapacityConfigType.ABSOLUTE_RESOURCE ? - maxGlobalPerQueueApps : - (int) (maxGlobalPerQueueApps * queueCapacities - .getAbsoluteCapacity(label)); - } else{ - maxApplications = (int) (conf.getMaximumSystemApplications() - * queueCapacities.getAbsoluteCapacity(label)); + for (String label : configuredNodeLabels) { + int maxApplicationsByLabel = 0; + if (maxGlobalPerQueueApps > 0) { + // In absolute mode, should + // shrink when change to corresponding label capacity. + maxApplicationsByLabel = this.capacityConfigType + != CapacityConfigType.ABSOLUTE_RESOURCE ? + maxGlobalPerQueueApps : + (int) (maxGlobalPerQueueApps * queueCapacities + .getAbsoluteCapacity(label)); + } else { + maxApplicationsByLabel = (int) (conf.getMaximumSystemApplications() + * queueCapacities.getAbsoluteCapacity(label)); + } + if (maxApplicationsByLabel > maxApplications) { + maxApplications = maxApplicationsByLabel; + maxLabel = label; + } } } leafQueue.setMaxApplications(maxApplications); @@ -1560,9 +1569,9 @@ void updateMaxAppRelatedField(CapacitySchedulerConfiguration conf, "update max app related, maxApplications=" + maxApplications + ", maxApplicationsPerUser=" + maxApplicationsPerUser + ", Abs Cap:" + queueCapacities - .getAbsoluteCapacity(label) + ", Cap: " + queueCapacities - .getCapacity(label) + ", MaxCap : " + queueCapacities - .getMaximumCapacity(label)); + .getAbsoluteCapacity(maxLabel) + ", Cap: " + queueCapacities + .getCapacity(maxLabel) + ", MaxCap : " + queueCapacities + .getMaximumCapacity(maxLabel)); } private void deriveCapacityFromAbsoluteConfigurations(String label, @@ -1643,11 +1652,6 @@ void updateEffectiveResources(Resource clusterResource) { deriveCapacityFromAbsoluteConfigurations(label, clusterResource, rc); // Re-visit max applications for a queue based on absolute capacity if // needed. - if (this instanceof LeafQueue) { - LeafQueue leafQueue = (LeafQueue) this; - CapacitySchedulerConfiguration conf = csContext.getConfiguration(); - updateMaxAppRelatedField(conf, leafQueue, label); - } } else{ queueResourceQuotas.setEffectiveMinResource(label, Resources .multiply(resourceByLabel, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index e5b41cede3a..a26fadf4342 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -632,6 +632,11 @@ public ConfiguredNodeLabels getConfiguredNodeLabels() { return configuredNodeLabels; } + @VisibleForTesting + public void reinitConfiguredNodeLabels(CapacitySchedulerConfiguration conf) { + this.configuredNodeLabels = new ConfiguredNodeLabels(conf); + } + private LeafQueue createAutoQueue(ApplicationPlacementContext queue) throws SchedulerDynamicEditException { List parentsToCreate = determineMissingParents(queue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 19de6db21b7..44727fb1db1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -1945,14 +1945,9 @@ public void updateClusterResource(Resource clusterResource, updateAbsoluteCapacities(); - // If maxApplications not set, use the system total max app, apply newly - // calculated abs capacity of the queue. - // When add new queue, the parent queue's other children should also - // update the max app. - super.updateMaxAppRelatedField(csContext.getConfiguration(), - this, CommonNodeLabelsManager.NO_LABEL); - super.updateEffectiveResources(clusterResource); + super.updateMaxAppRelatedField(csContext.getConfiguration(), + this); updateCurrentResourceLimits(currentResourceLimits, clusterResource); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index d5c7bc58ca8..c784b19865e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -141,7 +141,7 @@ public void testAutoCreateLeafQueueCreation() throws Exception { validateInitialQueueEntitlement(parentQueue, USER0, expectedChildQueueAbsCapacity, accessibleNodeLabelsOnC); - validateUserAndAppLimits(autoCreatedLeafQueue, 1000, 1000); + validateUserAndAppLimits(autoCreatedLeafQueue, 4000, 4000); validateContainerLimits(autoCreatedLeafQueue); assertTrue(autoCreatedLeafQueue @@ -911,7 +911,7 @@ public void testReinitializeQueuesWithAutoCreatedLeafQueues() AutoCreatedLeafQueue user0Queue = (AutoCreatedLeafQueue) newCS.getQueue( USER1); validateCapacities(user0Queue, 0.5f, 0.15f, 1.0f, 0.5f); - validateUserAndAppLimits(user0Queue, 1500, 1500); + validateUserAndAppLimits(user0Queue, 4000, 4000); //update leaf queue template capacities conf.setAutoCreatedLeafQueueConfigCapacity(C, 30f); @@ -919,7 +919,7 @@ public void testReinitializeQueuesWithAutoCreatedLeafQueues() newCS.reinitialize(conf, newMockRM.getRMContext()); validateCapacities(user0Queue, 0.3f, 0.09f, 0.4f, 0.2f); - validateUserAndAppLimits(user0Queue, 900, 900); + validateUserAndAppLimits(user0Queue, 4000, 4000); //submit app1 as USER3 submitApp(newMockRM, parentQueue, USER3, USER3, 3, 1); @@ -927,7 +927,7 @@ public void testReinitializeQueuesWithAutoCreatedLeafQueues() (AutoCreatedLeafQueue) newCS.getQueue(USER1); validateCapacities(user3Queue, 0.3f, 0.09f, 0.4f,0.2f); - validateUserAndAppLimits(user3Queue, 900, 900); + validateUserAndAppLimits(user3Queue, 4000, 4000); //submit app1 as USER1 - is already activated. there should be no diff // in capacities @@ -935,7 +935,7 @@ public void testReinitializeQueuesWithAutoCreatedLeafQueues() validateCapacities(user3Queue, 0.3f, 0.09f, 0.4f,0.2f); - validateUserAndAppLimits(user3Queue, 900, 900); + validateUserAndAppLimits(user3Queue, 4000, 4000); validateContainerLimits(user3Queue); GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index b6cc132da53..7a310138da8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -5120,6 +5120,45 @@ private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { return queue; } + @Test + public void testMaxApplicationsWithNodeLabels() throws IOException { + CapacitySchedulerConfiguration conf = csConf; + String rootChild = root.getChildQueues().get(0).getQueuePath(); + + conf.setCapacityByLabel(ROOT, "test", 100); + conf.setCapacityByLabel(rootChild, "test", 100); + conf.setCapacityByLabel(rootChild + "." + A, "test", 20); + conf.setCapacityByLabel(rootChild + "." + B, "test", 40); + conf.setCapacityByLabel(rootChild + "." + C, "test", 10); + conf.setCapacityByLabel(rootChild + "." + C + "." + C1, "test", 100); + conf.setCapacityByLabel(rootChild + "." + D, "test", 30); + conf.setCapacityByLabel(rootChild + "." + E, "test", 0); + cs.getCapacitySchedulerQueueManager().reinitConfiguredNodeLabels(conf); + cs.setMaxRunningAppsEnforcer(new CSMaxRunningAppsEnforcer(cs)); + cs.reinitialize(conf, cs.getRMContext()); + + LeafQueue e = (LeafQueue) cs.getQueue("e"); + // Maximum application should be calculated with the default node label + Assert.assertEquals("Maximum application is not calculated properly", + (int)(conf.getMaximumSystemApplications() + * e.getAbsoluteCapacity()), e.getMaxApplications()); + + conf.setCapacityByLabel(rootChild + "." + A, "test", 10); + conf.setCapacityByLabel(rootChild + "." + B, "test", 10); + conf.setCapacityByLabel(rootChild + "." + C, "test", 10); + conf.setCapacityByLabel(rootChild + "." + D, "test", 10); + conf.setCapacityByLabel(rootChild + "." + E, "test", 60); + cs.reinitialize(conf, cs.getRMContext()); + + e = (LeafQueue) cs.getQueue("e"); + // Maximum application is now determined by test label, because that would + // yield a higher value than with default node label + Assert.assertEquals("Maximum application is not calculated properly", + (int)(conf.getMaximumSystemApplications() * + e.getQueueCapacities().getAbsoluteCapacity("test")), + e.getMaxApplications()); + } + @After public void tearDown() throws Exception { if (cs != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 1dec2da113e..1ba064f7b7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -141,7 +141,7 @@ Configuration | Property | Description | |:---- |:---- | -| `yarn.scheduler.capacity.maximum-applications` / `yarn.scheduler.capacity..maximum-applications` | Maximum number of applications in the system which can be concurrently active both running and pending. Limits on each queue are directly proportional to their queue capacities and user limits. This is a hard limit and any applications submitted when this limit is reached will be rejected. Default is 10000. This can be set for all queues with `yarn.scheduler.capacity.maximum-applications` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity..maximum-applications`. Integer value expected. | +| `yarn.scheduler.capacity.maximum-applications` / `yarn.scheduler.capacity..maximum-applications` | Maximum number of applications in the system which can be concurrently active both running and pending. Limits on each queue are directly proportional to their queue capacities and user limits. This is a hard limit and any applications submitted when this limit is reached will be rejected. Default is 10000. This can be set for all queues with `yarn.scheduler.capacity.maximum-applications` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity..maximum-applications`. When this property is not set for a specific queue path, the maximum application number is calculated by taking all configured node labels into consideration, and choosing the highest possible value. Integer value expected. | | `yarn.scheduler.capacity.maximum-am-resource-percent` / `yarn.scheduler.capacity..maximum-am-resource-percent` | Maximum percent of resources in the cluster which can be used to run application masters - controls number of concurrent active applications. Limits on each queue are directly proportional to their queue capacities and user limits. Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with `yarn.scheduler.capacity.maximum-am-resource-percent` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity..maximum-am-resource-percent` | | `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity..max-parallel-apps`. Integer value is expected. By default, there is no limit. |