From 940747420f4e918de77f903ce0f4d36b0655e6fe Mon Sep 17 00:00:00 2001 From: Sunil G Date: Tue, 24 Oct 2017 00:30:46 +0530 Subject: [PATCH] YARN-7332.YARN-5881 --- .../scheduler/capacity/ParentQueue.java | 60 +++++++++++++++++++--- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 49fd4937a12..e8efbef73a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.AccessType; @@ -68,7 +69,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSetUtils; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; @Private @@ -927,24 +930,25 @@ private void calculateEffectiveResourcesAndCapacity(String label, // Factor to scale down effective resource: When cluster has sufficient // resources, effective_min_resources will be same as configured // min_resources. - float effectiveMinRatio = 1; + Resource numeratorForMinRatio = null; ResourceCalculator rc = this.csContext.getResourceCalculator(); if (getQueueName().equals("root")) { if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(rc, clusterResource, resourceByLabel, configuredMinResources)) { - effectiveMinRatio = Resources.divide(rc, clusterResource, - resourceByLabel, configuredMinResources); + numeratorForMinRatio = resourceByLabel; } } else { if (Resources.lessThan(rc, clusterResource, queueResourceQuotas.getEffectiveMinResource(label), configuredMinResources)) { - effectiveMinRatio = Resources.divide(rc, clusterResource, - queueResourceQuotas.getEffectiveMinResource(label), - configuredMinResources); + numeratorForMinRatio = queueResourceQuotas + .getEffectiveMinResource(label); } } + Map effectiveMinRatioPerResource = getEffectiveMinRatioPerResource( + configuredMinResources, numeratorForMinRatio); + // loop and do this for all child queues for (CSQueue childQueue : getChildQueues()) { Resource minResource = childQueue.getQueueResourceQuotas() @@ -954,7 +958,8 @@ private void calculateEffectiveResourcesAndCapacity(String label, if (childQueue.getCapacityConfigType() .equals(CapacityConfigType.ABSOLUTE_RESOURCE)) { childQueue.getQueueResourceQuotas().setEffectiveMinResource(label, - Resources.multiply(minResource, effectiveMinRatio)); + getMinResourceNormalized(effectiveMinRatioPerResource, + minResource)); // Max resource of a queue should be a minimum of {configuredMaxRes, // parentMaxRes}. parentMaxRes could be configured value. But if not @@ -1002,6 +1007,47 @@ private void calculateEffectiveResourcesAndCapacity(String label, } } + private Resource getMinResourceNormalized(Map effectiveMinRatio, + Resource minResource) { + Resource ret = Resource.newInstance(minResource); + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { + ResourceInformation nResourceInformation = minResource + .getResourceInformation(i); + + Float ratio = effectiveMinRatio.get(nResourceInformation.getName()); + if (ratio != null) { + ret.setResourceValue(i, + (long) (nResourceInformation.getValue() * ratio.floatValue())); + } + } + return ret; + } + + private Map getEffectiveMinRatioPerResource( + Resource configuredMinResources, Resource numeratorForMinRatio) { + Map effectiveMinRatioPerResource = new HashMap<>(); + if (numeratorForMinRatio != null) { + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { + ResourceInformation nResourceInformation = numeratorForMinRatio + .getResourceInformation(i); + ResourceInformation dResourceInformation = configuredMinResources + .getResourceInformation(i); + + long nValue = nResourceInformation.getValue(); + long dValue = UnitsConversionUtil.convert( + dResourceInformation.getUnits(), nResourceInformation.getUnits(), + dResourceInformation.getValue()); + if (dValue != 0) { + effectiveMinRatioPerResource.put(nResourceInformation.getName(), + (float) nValue / dValue); + } + } + } + return effectiveMinRatioPerResource; + } + private void deriveCapacityFromAbsoluteConfigurations(String label, Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) { -- 2.13.5 (Apple Git-94)