From 05aedb2503ec9ba3055a8075f8b2c5b735ec0790 Mon Sep 17 00:00:00 2001 From: liubin04 Date: Tue, 5 Jul 2022 11:12:57 +0800 Subject: [PATCH] YARN-11205. Do not reset max applications for a queue based on absolute capacity --- .../scheduler/capacity/ParentQueue.java | 19 ---------- .../scheduler/capacity/TestParentQueue.java | 36 +++++++++++++++++++ 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index d814a6054bc..3b05f2c6672 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -1117,25 +1117,6 @@ private void deriveCapacityFromAbsoluteConfigurations(String label, childQueue.getQueueCapacities().getMaximumCapacity(label) * getQueueCapacities().getAbsoluteMaximumCapacity(label)); - // Re-visit max applications for a queue based on absolute capacity if - // needed. - if (childQueue instanceof LeafQueue) { - LeafQueue leafQueue = (LeafQueue) childQueue; - CapacitySchedulerConfiguration conf = csContext.getConfiguration(); - int maxApplications = (int) (conf.getMaximumSystemApplications() - * childQueue.getQueueCapacities().getAbsoluteCapacity(label)); - leafQueue.setMaxApplications(maxApplications); - - int maxApplicationsPerUser = Math.min(maxApplications, - (int) (maxApplications - * (leafQueue.getUsersManager().getUserLimit() / 100.0f) - * leafQueue.getUsersManager().getUserLimitFactor())); - leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser); - LOG.info("LeafQueue:" + leafQueue.getQueuePath() + ", maxApplications=" - + maxApplications + ", maxApplicationsPerUser=" - + maxApplicationsPerUser + ", Abs Cap:" - + childQueue.getQueueCapacities().getAbsoluteCapacity(label)); - } } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 0560d595a63..9800f62e029 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -1029,6 +1030,41 @@ public void testAbsoluteResourceWithChangeInClusterResource() QUEUE_B_RESOURCE_70PERC); } + @Test + public void testMaxApplicationsForAbsoluteResource() + throws Exception { + // Setup queue configs + setupSingleLevelQueuesWithAbsoluteResource(csConf); + csConf.setGlobalMaximumApplicationsPerQueue(100); + + CSQueueStore queues = new CSQueueStore(); + CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, + null, CapacitySchedulerConfiguration.ROOT, queues, queues, + TestUtils.spyHook); + + // Setup some nodes + final int memoryPerNode = 10; + int coresPerNode = 16; + int numNodes = 2; + + Resource clusterResource = Resources.createResource( + numNodes * (memoryPerNode * GB), numNodes * coresPerNode); + when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + + // Start testing + LeafQueue a = (LeafQueue) queues.get(A); + LeafQueue b = (LeafQueue) queues.get(B); + + assertEquals(0.3, a.getQueueCapacities().getAbsoluteCapacity(), + EPSILON); + assertEquals(0.7, b.getQueueCapacities().getAbsoluteCapacity(), + EPSILON); + assertEquals(100, a.getMaxApplications()); + assertEquals(100, b.getMaxApplications()); + } + @After public void tearDown() throws Exception { } -- 2.24.3 (Apple Git-128)