From 766d5ca97d35fb5023ccbc0515f9c44d4989164c Mon Sep 17 00:00:00 2001 From: Sunil G Date: Fri, 3 Nov 2017 07:31:49 +0530 Subject: [PATCH] YARN-7441.YARN-5881 --- .../util/resource/DefaultResourceCalculator.java | 8 ++++++++ .../util/resource/DominantResourceCalculator.java | 21 +++++++++++++++++++ .../yarn/util/resource/ResourceCalculator.java | 14 ++++++++++++- .../hadoop/yarn/util/resource/Resources.java | 5 +++++ .../AbstractPreemptableResourceCalculator.java | 24 +++++++++++++++++++--- .../monitor/capacity/TempQueuePerPartition.java | 12 +++++++---- 6 files changed, 76 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java index aefa85cb709..6375c4afd9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java @@ -112,6 +112,14 @@ public Resource multiplyAndNormalizeUp(Resource r, double by, } @Override + public Resource multiplyAndNormalizeUp(Resource r, double[] by, + Resource stepFactor) { + return Resources.createResource( + roundUp((long) (r.getMemorySize() * by[0] + 0.5), + stepFactor.getMemorySize())); + } + + @Override public Resource multiplyAndNormalizeDown(Resource r, double by, Resource stepFactor) { return Resources.createResource( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index ac3bfdcdc93..8ef155e6a7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -496,6 +496,27 @@ private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) { } @Override + public Resource multiplyAndNormalizeUp(Resource r, double[] by, + Resource stepFactor) { + Resource ret = Resource.newInstance(r); + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { + ResourceInformation rResourceInformation = r.getResourceInformation(i); + ResourceInformation stepFactorResourceInformation = stepFactor + .getResourceInformation(i); + + long rValue = rResourceInformation.getValue(); + long stepFactorValue = UnitsConversionUtil.convert( + stepFactorResourceInformation.getUnits(), + rResourceInformation.getUnits(), + stepFactorResourceInformation.getValue()); + ret.setResourceValue(i, ResourceCalculator + .roundUp((long) Math.ceil(rValue * by[i]), stepFactorValue)); + } + return ret; + } + + @Override public Resource multiplyAndNormalizeUp(Resource r, double by, Resource stepFactor) { return this.multiplyAndNormalize(r, by, stepFactor, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index a8162906684..dfd6684322d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -125,7 +125,19 @@ public abstract long computeAvailableContainers( */ public abstract Resource multiplyAndNormalizeUp( Resource r, double by, Resource stepFactor); - + + /** + * Multiply resource r by factor by + * and normalize up using step-factor stepFactor. + * + * @param r resource to be multiplied + * @param by multiplier array for all resource types + * @param stepFactor factor by which to normalize up + * @return resulting normalized resource + */ + public abstract Resource multiplyAndNormalizeUp( + Resource r, double[] by, Resource stepFactor); + /** * Multiply resource r by factor by * and normalize down using step-factor stepFactor. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 70c2e731725..ff6d1639a39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -295,6 +295,11 @@ public static Resource multiplyAndAddTo( return lhs; } + public static Resource multiplyAndNormalizeUp(ResourceCalculator calculator, + Resource lhs, double[] by, Resource factor) { + return calculator.multiplyAndNormalizeUp(lhs, by, factor); + } + public static Resource multiplyAndNormalizeUp( ResourceCalculator calculator,Resource lhs, double by, Resource factor) { return calculator.multiplyAndNormalizeUp(lhs, by, factor); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java index a80f317bb4c..5196831e2ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java @@ -19,8 +19,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.PriorityUtilizationQueueOrderingPolicy; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import java.util.ArrayList; @@ -198,18 +201,33 @@ protected void computeFixpointAllocation(Resource totGuarant, private void resetCapacity(Resource clusterResource, Collection queues, boolean ignoreGuar) { Resource activeCap = Resource.newInstance(0, 0); + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); if (ignoreGuar) { for (TempQueuePerPartition q : queues) { - q.normalizedGuarantee = 1.0f / queues.size(); + for (int i = 0; i < maxLength; i++) { + q.normalizedGuarantee[i] = 1.0f / queues.size(); + } } } else { for (TempQueuePerPartition q : queues) { Resources.addTo(activeCap, q.getGuaranteed()); } for (TempQueuePerPartition q : queues) { - q.normalizedGuarantee = Resources.divide(rc, clusterResource, - q.getGuaranteed(), activeCap); + for (int i = 0; i < maxLength; i++) { + ResourceInformation nResourceInformation = q.getGuaranteed() + .getResourceInformation(i); + ResourceInformation dResourceInformation = activeCap + .getResourceInformation(i); + + long nValue = nResourceInformation.getValue(); + long dValue = UnitsConversionUtil.convert( + dResourceInformation.getUnits(), nResourceInformation.getUnits(), + dResourceInformation.getValue()); + if (dValue != 0) { + q.normalizedGuarantee[i] = (float) nValue / dValue; + } + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java index bd236febbc0..4d71223aaa3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java @@ -22,9 +22,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; @@ -46,7 +48,7 @@ Resource untouchableExtra; Resource preemptableExtra; - double normalizedGuarantee; + double[] normalizedGuarantee; private Resource effMinRes; private Resource effMaxRes; @@ -88,7 +90,8 @@ pendingDeductReserved = Resources.createResource(0); } - this.normalizedGuarantee = Float.NaN; + this.normalizedGuarantee = new double[ResourceUtils + .getNumberOfKnownResourceTypes()]; this.children = new ArrayList<>(); this.apps = new ArrayList<>(); this.untouchableExtra = Resource.newInstance(0, 0); @@ -240,8 +243,9 @@ public String toString() { sb.append(" NAME: " + queueName).append(" CUR: ").append(current) .append(" PEN: ").append(pending).append(" RESERVED: ").append(reserved) .append(" GAR: ").append(getGuaranteed()).append(" NORM: ") - .append(normalizedGuarantee).append(" IDEAL_ASSIGNED: ") - .append(idealAssigned).append(" IDEAL_PREEMPT: ").append(toBePreempted) + .append(Arrays.toString(normalizedGuarantee)) + .append(" IDEAL_ASSIGNED: ").append(idealAssigned) + .append(" IDEAL_PREEMPT: ").append(toBePreempted) .append(" ACTUAL_PREEMPT: ").append(getActuallyToBePreempted()) .append(" UNTOUCHABLE: ").append(untouchableExtra) .append(" PREEMPTABLE: ").append(preemptableExtra).append("\n"); -- 2.13.5 (Apple Git-94)