diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 2aa03a0ba59..9f606703f1d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.yarn.util.resource; +import java.util.Iterator; +import java.util.NavigableSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -27,6 +29,7 @@ import org.apache.hadoop.yarn.util.UnitsConversionUtil; import java.util.Set; +import java.util.TreeSet; /** * A {@link ResourceCalculator} which uses the concept of @@ -115,39 +118,77 @@ public int compare(Resource clusterResource, Resource lhs, Resource rhs, return this.compare(lhs, rhs); } - float l = getResourceAsValue(clusterResource, lhs, true); - float r = getResourceAsValue(clusterResource, rhs, true); - - if (l < r) { - return -1; - } else if (l > r) { - return 1; - } else if (!singleType) { - l = getResourceAsValue(clusterResource, lhs, false); - r = getResourceAsValue(clusterResource, rhs, false); - - if (l < r) { - return -1; - } else if (l > r) { - return 1; + // We have to calculate the shares for all resource types for both + // resources, sort them, and then look for which resource has the bigger + // share overall. + NavigableSet lhsShares = new TreeSet<>(); + NavigableSet rhsShares = new TreeSet<>(); + + for (String rName : resourceNames) { + ResourceInformation clusterResourceInfo; + ResourceInformation lhsResourceInfo; + ResourceInformation rhsResourceInfo; + + try { + clusterResourceInfo = clusterResource.getResourceInformation(rName); + } catch (YarnException ye) { + throw new IllegalArgumentException( + "Error getting resource information for " + clusterResource, ye); + } + try { + lhsResourceInfo = lhs.getResourceInformation(rName); + } catch (YarnException ye) { + throw new IllegalArgumentException( + "Error getting resource information for " + lhs, ye); + } + try { + rhsResourceInfo = rhs.getResourceInformation(rName); + } catch (YarnException ye) { + throw new IllegalArgumentException( + "Error getting resource information for " + rhs, ye); + } + + // Convert the resources' units into the cluster resource's units + long lhsResourceValue = + UnitsConversionUtil.convert(lhsResourceInfo.getUnits(), + clusterResourceInfo.getUnits(), lhsResourceInfo.getValue()); + long rhsResourceValue = + UnitsConversionUtil.convert(lhsResourceInfo.getUnits(), + clusterResourceInfo.getUnits(), rhsResourceInfo.getValue()); + + lhsShares.add((double) lhsResourceValue / clusterResourceInfo.getValue()); + rhsShares.add((double) rhsResourceValue / clusterResourceInfo.getValue()); + } + + Iterator lhsIter = lhsShares.descendingIterator(); + Iterator rhsIter = rhsShares.descendingIterator(); + double ret = 0.0; + + while (lhsIter.hasNext() && rhsIter.hasNext()) { + ret = lhsIter.next() - rhsIter.next(); + + if (ret != 0.0) { + break; } } - return 0; + return (int)Math.signum(ret); } /** - * Use 'dominant' for now since we only have 2 resources - gives us a slight - * performance boost. - *

- * Once we add more resources, we'll need a more complicated (and slightly - * less performant algorithm). + * Return share of the resource type that has the largest share for the + * resource. The share is calculated as a percentage of the total amount + * of each resource type available in the cluster. + * @param clusterResource a {{Resource}} that represents to total resources + * available in the cluster + * @param resource a {{Resource}} to compare against the cluster resources + * @return the share of the resource type for which the given resource has + * the largest share */ - protected float getResourceAsValue(Resource clusterResource, - Resource resource, boolean dominant) { - - float min = Float.MAX_VALUE; + private float getResourceAsDominantValue(Resource clusterResource, + Resource resource) { float max = 0.0f; + for (String rName : resourceNames) { try { ResourceInformation clusterResourceResourceInformation = @@ -161,14 +202,15 @@ protected float getResourceAsValue(Resource clusterResource, float tmp = (float) resourceValue / (float) clusterResourceResourceInformation .getValue(); - min = min < tmp ? min : tmp; + max = max > tmp ? max : tmp; } catch (YarnException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); } } - return (dominant) ? max : min; + + return max; } @Override @@ -200,8 +242,8 @@ public long computeAvailableContainers(Resource available, Resource required) { public float divide(Resource clusterResource, Resource numerator, Resource denominator) { return - getResourceAsValue(clusterResource, numerator, true) / - getResourceAsValue(clusterResource, denominator, true); + getResourceAsDominantValue(clusterResource, numerator) / + getResourceAsDominantValue(clusterResource, denominator); } @Override