From cc7921772ad3ed897df648202c3743d018926a7c Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 22 Apr 2020 23:17:01 +0530 Subject: [PATCH] YARN-10154. Addendum Patch to support Absolute Resource in AutoCreatedLeafQueue --- .../scheduler/capacity/ManagedParentQueue.java | 89 ++++++++++++---------- 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index 2e0e4dd..358bbdd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -183,45 +183,6 @@ private void reinitializeQueueManagementPolicy() throws IOException { CSQueueUtils.loadUpdateAndCheckCapacities(csContext.getConfiguration() .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), csContext.getConfiguration(), queueCapacities, getQueueCapacities()); - - - /** - * Populate leaf queue template (of Parent resources configured in - * ABSOLUTE_RESOURCE) capacities with actual values for which configured has - * been defined in ABSOLUTE_RESOURCE format. - * - */ - if (this.capacityConfigType.equals(CapacityConfigType.ABSOLUTE_RESOURCE)) { - for (String label : queueCapacities.getExistingNodeLabels()) { - queueCapacities.setCapacity(label, - this.csContext.getResourceCalculator().divide( - this.csContext.getClusterResource(), - this.csContext.getConfiguration().getMinimumResourceRequirement( - label, - this.csContext.getConfiguration() - .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), - resourceTypes), - getQueueResourceQuotas().getConfiguredMinResource(label))); - - queueCapacities.setMaximumCapacity(label, - this.csContext.getResourceCalculator().divide( - this.csContext.getClusterResource(), - this.csContext.getConfiguration().getMaximumResourceRequirement( - label, - this.csContext.getConfiguration() - .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), - resourceTypes), - getQueueResourceQuotas().getConfiguredMaxResource(label))); - - queueCapacities.setAbsoluteCapacity(label, - queueCapacities.getCapacity(label) - * getQueueCapacities().getAbsoluteCapacity(label)); - - queueCapacities.setAbsoluteMaximumCapacity(label, - queueCapacities.getMaximumCapacity(label) - * getQueueCapacities().getAbsoluteMaximumCapacity(label)); - } - } builder.capacities(queueCapacities); return builder; } @@ -279,6 +240,56 @@ public void addChildQueue(CSQueue childQueue) final AutoCreatedLeafQueueConfig initialLeafQueueTemplate = queueManagementPolicy.getInitialLeafQueueConfiguration(leafQueue); + /** + * Populate leaf queue template (of Parent resources configured in + * ABSOLUTE_RESOURCE) capacities with actual values for which configured has + * been defined in ABSOLUTE_RESOURCE format. + * + */ + if (this.capacityConfigType.equals(CapacityConfigType.ABSOLUTE_RESOURCE)) { + QueueCapacities queueCapacities = initialLeafQueueTemplate.getQueueCapacities(); + for (String label : queueCapacities.getExistingNodeLabels()) { + queueCapacities.setCapacity(label, + this.csContext.getResourceCalculator().divide( + this.csContext.getClusterResource(), + this.csContext.getConfiguration().getMinimumResourceRequirement( + label, + this.csContext.getConfiguration() + .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), + resourceTypes), + getQueueResourceQuotas().getConfiguredMinResource(label))); + + Resource childMaxResource = this.csContext.getConfiguration() + .getMaximumResourceRequirement(label, + this.csContext.getConfiguration() + .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), + resourceTypes); + Resource parentMaxRes = getQueueResourceQuotas() + .getConfiguredMaxResource(label); + + Resource effMaxResource = Resources.min( + this.csContext.getResourceCalculator(), + this.csContext.getClusterResource(), + childMaxResource.equals(Resources.none()) ? parentMaxRes + : childMaxResource, + parentMaxRes); + + queueCapacities.setMaximumCapacity( + label, this.csContext.getResourceCalculator().divide( + this.csContext.getClusterResource(), + effMaxResource, + getQueueResourceQuotas().getConfiguredMaxResource(label))); + + queueCapacities.setAbsoluteCapacity( + label, queueCapacities.getCapacity(label) + * getQueueCapacities().getAbsoluteCapacity(label)); + + queueCapacities.setAbsoluteMaximumCapacity(label, + queueCapacities.getMaximumCapacity(label) + * getQueueCapacities().getAbsoluteMaximumCapacity(label)); + } + } + leafQueue.reinitializeFromTemplate(initialLeafQueueTemplate); } finally { writeLock.unlock(); -- 2.7.4 (Apple Git-66)