diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java index 9e5bdb0bc0b..3b12360db83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler .SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common @@ -67,6 +68,22 @@ public void setEntitlement(QueueEntitlement entitlement) setEntitlement(NO_LABEL, entitlement); } + @Override + protected Resource getMinimumAbsoluteResource(String queuePath, + String label) { + return super.getMinimumAbsoluteResource(csContext.getConfiguration() + .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), + label); + } + + @Override + protected Resource getMaximumAbsoluteResource(String queuePath, + String label) { + return super.getMaximumAbsoluteResource(csContext.getConfiguration() + .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), + label); + } + /** * This methods to change capacity for a queue and adjusts its * absoluteCapacity diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 4345cbd5a3f..48283222aa4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -514,16 +514,26 @@ private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) { return unionInheritedWeights; } + protected Resource getMinimumAbsoluteResource(String queuePath, String label) { + Resource minResource = csContext.getConfiguration() + .getMinimumResourceRequirement(label, queuePath, resourceTypes); + return minResource; + } + + protected Resource getMaximumAbsoluteResource(String queuePath, String label) { + Resource maxResource = csContext.getConfiguration() + .getMaximumResourceRequirement(label, queuePath, resourceTypes); + return maxResource; + } + protected void updateConfigurableResourceRequirement(String queuePath, Resource clusterResource) { CapacitySchedulerConfiguration conf = csContext.getConfiguration(); Set configuredNodelabels = conf.getConfiguredNodeLabels(queuePath); for (String label : configuredNodelabels) { - Resource minResource = conf.getMinimumResourceRequirement(label, - queuePath, resourceTypes); - Resource maxResource = conf.getMaximumResourceRequirement(label, - queuePath, resourceTypes); + Resource minResource = getMinimumAbsoluteResource(queuePath, label); + Resource maxResource = getMaximumAbsoluteResource(queuePath, label); LOG.debug("capacityConfigType is '{}' for queue {}", capacityConfigType, getQueueName()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index d57e3919cdd..4c2a3e97f25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -556,6 +556,12 @@ public void setMaximumCapacityByLabel(String queue, String label, float capacity) { setFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY, capacity); } + + public void setMaximumCapacityByLabel(String queue, String label, + String absoluteResourceCapacity) { + set(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY, + absoluteResourceCapacity); + } public int getUserLimit(String queue) { int userLimit = getInt(getQueuePrefix(queue) + USER_LIMIT, @@ -1953,11 +1959,29 @@ public void setAutoCreatedLeafQueueConfigCapacity(String queuePath, @Private public void setAutoCreatedLeafQueueTemplateCapacityByLabel(String queuePath, String label, float val) { - String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix( - queuePath); + String leafQueueConfPrefix = + getAutoCreatedQueueTemplateConfPrefix(queuePath); setCapacityByLabel(leafQueueConfPrefix, label, val); } + @VisibleForTesting + @Private + public void setAutoCreatedLeafQueueTemplateCapacityByLabel(String queuePath, + String label, Resource resource) { + + String leafQueueConfPrefix = + getAutoCreatedQueueTemplateConfPrefix(queuePath); + + StringBuilder resourceString = new StringBuilder(); + resourceString + .append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "=" + + resource.getMemorySize() + "," + + AbsoluteResourceType.VCORES.toString().toLowerCase() + "=" + + resource.getVirtualCores() + "]"); + + setCapacityByLabel(leafQueueConfPrefix, label, resourceString.toString()); + } + @Private @VisibleForTesting public void setAutoCreatedLeafQueueConfigMaxCapacity(String queuePath, @@ -1976,6 +2000,23 @@ public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath, setMaximumCapacityByLabel(leafQueueConfPrefix, label, val); } + @Private + @VisibleForTesting + public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath, + String label, Resource resource) { + String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix( + queuePath); + + StringBuilder resourceString = new StringBuilder(); + resourceString + .append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "=" + + resource.getMemorySize() + "," + + AbsoluteResourceType.VCORES.toString().toLowerCase() + "=" + + resource.getVirtualCores() + "]"); + + setMaximumCapacityByLabel(leafQueueConfPrefix, label, resourceString.toString()); + } + @VisibleForTesting @Private public void setAutoCreatedLeafQueueConfigUserLimit(String queuePath, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index fe4c880a56a..3cde569cd67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.junit.Assert; import org.junit.Test; @@ -38,6 +39,7 @@ private static final String QUEUEA = "queueA"; private static final String QUEUEB = "queueB"; private static final String QUEUEC = "queueC"; + private static final String QUEUED = "queueD"; private static final String QUEUEA1 = "queueA1"; private static final String QUEUEA2 = "queueA2"; private static final String QUEUEB1 = "queueB1"; @@ -48,6 +50,9 @@ + "." + QUEUEB; private static final String QUEUEC_FULL = CapacitySchedulerConfiguration.ROOT + "." + QUEUEC; + private static final String QUEUED_FULL = CapacitySchedulerConfiguration.ROOT + + "." + QUEUED; + private static final String QUEUEA1_FULL = QUEUEA_FULL + "." + QUEUEA1; private static final String QUEUEA2_FULL = QUEUEA_FULL + "." + QUEUEA2; private static final String QUEUEB1_FULL = QUEUEB_FULL + "." + QUEUEB1; @@ -66,13 +71,17 @@ 10); private static final Resource QUEUE_B_MAXRES = Resource.newInstance(150 * GB, 30); - private static final Resource QUEUE_C_MINRES = Resource.newInstance(50 * GB, - 10); + private static final Resource QUEUE_C_MINRES = Resource.newInstance(25 * GB, + 5); private static final Resource QUEUE_C_MAXRES = Resource.newInstance(150 * GB, 20); + private static final Resource QUEUE_D_MINRES = Resource.newInstance(25 * GB, + 5); + private static final Resource QUEUE_D_MAXRES = Resource.newInstance(150 * GB, + 20); private static final Resource QUEUEA_REDUCED = Resource.newInstance(64000, 6); private static final Resource QUEUEB_REDUCED = Resource.newInstance(32000, 6); - private static final Resource QUEUEC_REDUCED = Resource.newInstance(32000, 6); + private static final Resource QUEUEC_REDUCED = Resource.newInstance(16000, 3); private static final Resource QUEUEMAX_REDUCED = Resource.newInstance(128000, 20); @@ -83,15 +92,24 @@ private CapacitySchedulerConfiguration setupSimpleQueueConfiguration( boolean isCapacityNeeded) { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[]{QUEUEA, QUEUEB, QUEUEC}); + new String[]{QUEUEA, QUEUEB, QUEUEC, QUEUED}); // Set default capacities like normal configuration. if (isCapacityNeeded) { csConf.setCapacity(QUEUEA_FULL, 50f); csConf.setCapacity(QUEUEB_FULL, 25f); csConf.setCapacity(QUEUEC_FULL, 25f); + csConf.setCapacity(QUEUED_FULL, 25f); } + csConf.setAutoCreateChildQueueEnabled(QUEUED_FULL, true); + + // Setup leaf queue template configs + csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUED_FULL, "", + QUEUE_D_MINRES); + csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUED_FULL, "", + QUEUE_D_MAXRES); + return csConf; } @@ -122,10 +140,12 @@ private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration( csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES); csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES); csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES); + csConf.setMinimumResourceRequirement("", QUEUED_FULL, QUEUE_D_MINRES); csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES); csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES); csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES); + csConf.setMaximumResourceRequirement("", QUEUED_FULL, QUEUE_D_MAXRES); return csConf; } @@ -148,7 +168,8 @@ private CapacitySchedulerConfiguration setupComplexMinMaxResourceConfig( } @Test - public void testSimpleMinMaxResourceConfigurartionPerQueue() { + public void testSimpleMinMaxResourceConfigurartionPerQueue() + throws Exception { CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(true); setupMinMaxResourceConfiguration(csConf); @@ -171,6 +192,69 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue() { Assert.assertEquals("Max resource configured for QUEUEC is not correct", QUEUE_C_MAXRES, csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes)); + + csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + @SuppressWarnings("resource") + MockRM rm = new MockRM(csConf); + rm.start(); + + // Add few nodes + rm.registerNode("127.0.0.1:1234", 250 * GB, 40); + + // Get queue object to verify min/max resource configuration. + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue(QUEUED); + AutoCreatedLeafQueue d1 = new AutoCreatedLeafQueue(cs, "d1", parentQueue); + cs.addQueue(d1); + + /** + * After adding child queue d1, it occupies all entire resource + * of Managed Parent queue + */ + cs.getRootQueue().updateClusterResource(cs.getClusterResource(), + new ResourceLimits(cs.getClusterResource())); + + Assert.assertEquals(QUEUE_D_MINRES, + d1.queueResourceQuotas.getConfiguredMinResource()); + Assert.assertEquals(QUEUE_D_MINRES, + d1.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals(QUEUE_D_MAXRES, + d1.queueResourceQuotas.getConfiguredMaxResource()); + Assert.assertEquals(QUEUE_D_MAXRES, + d1.queueResourceQuotas.getEffectiveMaxResource()); + + /** + * After adding child queue d2, adjustment happens and both d1 and d2 shares + * resource of Managed Parent Queue + */ + AutoCreatedLeafQueue d2 = new AutoCreatedLeafQueue(cs, "d2", parentQueue); + cs.addQueue(d2); + + cs.getRootQueue().updateClusterResource(cs.getClusterResource(), + new ResourceLimits(cs.getClusterResource())); + + Assert.assertEquals(QUEUE_D_MINRES, + d2.queueResourceQuotas.getConfiguredMinResource()); + Assert.assertEquals(Resource.newInstance(12800, 2), + d2.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals(QUEUE_D_MAXRES, + d2.queueResourceQuotas.getConfiguredMaxResource()); + Assert.assertEquals(QUEUE_D_MAXRES, + d2.queueResourceQuotas.getEffectiveMaxResource()); + + Assert.assertEquals(QUEUE_D_MINRES, + d1.queueResourceQuotas.getConfiguredMinResource()); + Assert.assertEquals(Resource.newInstance(12800, 2), + d1.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals(QUEUE_D_MAXRES, + d1.queueResourceQuotas.getConfiguredMaxResource()); + Assert.assertEquals(QUEUE_D_MAXRES, + d1.queueResourceQuotas.getEffectiveMaxResource()); + + rm.close(); } @Test @@ -576,7 +660,7 @@ public void testEffectiveResourceAfterIncreasingClusterResource() Assert.assertEquals("Effective Max resource for QUEUEC is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEC is not correct", - 0.2, qC.getAbsoluteCapacity(), DELTA); + 0.1, qC.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct", 0.6, qC.getAbsoluteMaximumCapacity(), DELTA); @@ -643,7 +727,7 @@ public void testEffectiveResourceAfterIncreasingClusterResource() Assert.assertEquals("Effective Max resource for QUEUEC is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEC is not correct", - 0.133, qC.getAbsoluteCapacity(), DELTA); + 0.066, qC.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct", 0.4, qC.getAbsoluteMaximumCapacity(), DELTA); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 5a463e65d13..7c20d85b159 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -377,10 +377,11 @@ The parent queue which has been enabled for auto leaf queue creation,supports | Property | Description | |:---- |:---- | -| `yarn.scheduler.capacity..leaf-queue-template.capacity` | *Mandatory* parameter: Specifies the minimum guaranteed capacity for the auto-created leaf queues. Currently *Absolute Resource* configurations are not supported on auto-created leaf queues | +| `yarn.scheduler.capacity..leaf-queue-template.capacity` | *Mandatory* parameter: Specifies the minimum guaranteed capacity for the auto-created leaf queues. | +| `yarn.scheduler.capacity..leaf-queue-template.maximum-capacity` | *Optional* parameter: Specifies the maximum capacity for the auto-created leaf queues. This value must be smaller than or equal to the cluster maximum. | | `yarn.scheduler.capacity..leaf-queue-template.` | *Optional* parameter: For other queue parameters that can be configured on auto-created leaf queues like maximum-capacity, user-limit-factor, maximum-am-resource-percent ... - Refer **Queue Properties** section | -Example: +Example 1: ``` @@ -421,6 +422,22 @@ Example: ``` +Example 2: + +``` + + yarn.scheduler.capacity.root.parent2.auto-create-child-queue.enabled + true + + + yarn.scheduler.capacity.root.parent2.leaf-queue-template.capacity + [memory=1024,vcores=1] + + + yarn.scheduler.capacity.root.parent2.leaf-queue-template.maximum-capacity + [memory=10240,vcores=10] + +``` * Scheduling Edit Policy configuration for auto-created queue management Admins need to specify an additional `org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueManagementDynamicEditPolicy` scheduling edit policy to the