diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index ea5c8a8c3f6..8c6ba39e7d8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.util.resource; +import java.util.Arrays; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -114,39 +115,165 @@ public int compare(Resource clusterResource, Resource lhs, Resource rhs, return this.compare(lhs, rhs); } - float l = getResourceAsValue(clusterResource, lhs, true); - float r = getResourceAsValue(clusterResource, rhs, true); - - if (l < r) { - return -1; - } else if (l > r) { - return 1; - } else if (!singleType) { - l = getResourceAsValue(clusterResource, lhs, false); - r = getResourceAsValue(clusterResource, rhs, false); - - if (l < r) { - return -1; - } else if (l > r) { - return 1; + // We have to calculate the shares for all resource types for both + // resources and then look for which resource has the biggest + // share overall. + ResourceInformation[] clusterRes = clusterResource.getResources(); + // If array creation shows up as a time sink, these arrays could be cached + // because they're always the same length. + double[] lhsShares = new double[clusterRes.length]; + double[] rhsShares = new double[clusterRes.length]; + double diff = calculateShares(clusterRes, lhs, rhs, lhsShares, + rhsShares, singleType); + + // If singleType is true, diff already contains our answer. If not, we still + // have to calculate it. + if (!singleType) { + diff = compareShares(lhsShares, rhsShares); + } + + return (int)diff; + } + + /** + * Calculate the shares for {@code lhs} and {@code rhs} according to + * {@code clusterRes}, and store the results in {@code lhsShares} and + * {@code rhsShares}, respectively. If {@code diffMax} is true, then + * return the difference between the maximum share from {@code lhsShares} and + * the maximum share from {@code rhsShare}. Otherwise return 0.0. + * @param clusterRes the array of ResourceInformation instances that + * represents the cluster's maximum resources + * @param lhs the first resource to compare + * @param rhs the second resource to compare + * @param lhsShares an array to store the shares for the first resource + * @param rhsShares an array to store the shares for the second resource + * @param diffMax whether to return the diff of the maximum shares + * @return -1.0, 0.0, or 1.0, depending on whether the max share of the first + * resource is less than, equal to, or greater than the max share of the + * second resource, respectively, or 0.0 if {@code maxDiff} is false + */ + private double calculateShares(ResourceInformation[] clusterRes, Resource lhs, + Resource rhs, double[] lhsShares, double[] rhsShares, + boolean diffMax) { + double[] max = null; + + if (diffMax) { + max = new double[2]; + } + + calculateShares(clusterRes, lhs, rhs, lhsShares, rhsShares, max); + + if (diffMax) { + return Math.signum(max[0] - max[1]); + } else { + return 0.0; + } + } + + /** + * Calculate the shares for {@code first} and {@code second} according to + * {@code clusterRes}, and store the results in {@code firstShares} and + * {@code secondShares}, respectively. If {@code max} is non-null, then + * {@cade max} will be populated with the max shares from {@code firstShare} + * and {@code secondShare} in the first and second indices, respectively. + * @param clusterRes the array of ResourceInformation instances that + * represents the cluster's maximum resources + * @param first the first resource to compare + * @param second the second resource to compare + * @param firstShares an array to store the shares for the first resource + * @param secondShares an array to store the shares for the second resource + * @param max an array to store the max shares of the first and second + * resources + * @return -1.0, 0.0, or 1.0, depending on whether the max share of the first + * resource is less than, equal to, or greater than the max share of the + * second resource, respectively, or 0.0 if {@code max} is null + */ + private void calculateShares(ResourceInformation[] clusterRes, Resource first, + Resource second, double[] firstShares, double[] secondShares, + double[] max) { + ResourceInformation[] firstRes = first.getResources(); + ResourceInformation[] secondRes = second.getResources(); + boolean doMax = (max != null) && (max.length >= 2); + + if (doMax) { + max[0] = 0.0; + max[1] = 0.0; + } + + for (int i = 0; i < clusterRes.length; i++) { + firstShares[i] = calculateShare(clusterRes[i], firstRes[i]); + secondShares[i] = calculateShare(clusterRes[i], secondRes[i]); + + // Only track the max if singleType is true + if (doMax) { + if (firstShares[i] > max[0]) { + max[0] = firstShares[i]; + } + + if (secondShares[i] > max[1]) { + max[1] = secondShares[i]; + } } } + } + + /** + * Calculate the share for a resource type. + * @param clusterRes the resource type for the cluster maximum + * @param res the resource type for which to calculate the share + * @return the share + */ + private double calculateShare(ResourceInformation clusterRes, + ResourceInformation res) { + // Convert the resources' units into the cluster resource's units + long value = UnitsConversionUtil.convert(res.getUnits(), + clusterRes.getUnits(), res.getValue()); - return 0; + return (double) value / clusterRes.getValue(); } /** - * Use 'dominant' for now since we only have 2 resources - gives us a slight - * performance boost. - *

- * Once we add more resources, we'll need a more complicated (and slightly - * less performant algorithm). + * Compare the two shares arrays by comparing the largest elements, then the + * next largest if the previous were equal, etc. This method sorts the shares + * arrays. + * @param lhsShares the first share array to compare + * @param rhsShares the second share array to compare + * @return -1.0, 0.0, or 1.0 depending whether the first array is less than, + * equal to, or greater than the second array */ - protected float getResourceAsValue(Resource clusterResource, - Resource resource, boolean dominant) { + private double compareShares(double[] lhsShares, double[] rhsShares) { + double diff = 0.0; + + Arrays.sort(lhsShares); + Arrays.sort(rhsShares); + + // lhsShares and rhsShares must necessarily have the same length, because + // everyone uses the same master resource list. + for (int i = 0; i < lhsShares.length; i++) { + diff = lhsShares[i] - rhsShares[i]; - float min = Float.MAX_VALUE; + if (diff != 0.0) { + break; + } + } + + return Math.signum(diff); + } + + /** + * Return share of the resource type that has the largest share for the + * resource. The share is calculated as a percentage of the total amount + * of each resource type available in the cluster. + * @param clusterResource a {{Resource}} that represents to total resources + * available in the cluster + * @param resource a {{Resource}} to compare against the cluster resources + * @return the share of the resource type for which the given resource has + * the largest share + */ + private float getResourceAsDominantValue(Resource clusterResource, + Resource resource) { float max = 0.0f; + for (String rName : resourceNames) { try { ResourceInformation clusterResourceResourceInformation = @@ -160,14 +287,17 @@ protected float getResourceAsValue(Resource clusterResource, float tmp = (float) resourceValue / (float) clusterResourceResourceInformation .getValue(); - min = min < tmp ? min : tmp; - max = max > tmp ? max : tmp; + + if (tmp > max) { + max = tmp; + } } catch (ResourceNotFoundException ye) { throw new IllegalArgumentException( "Error getting resource information for " + resource, ye); } } - return (dominant) ? max : min; + + return max; } @Override @@ -198,9 +328,19 @@ public long computeAvailableContainers(Resource available, Resource required) { @Override public float divide(Resource clusterResource, Resource numerator, Resource denominator) { - return - getResourceAsValue(clusterResource, numerator, true) / - getResourceAsValue(clusterResource, denominator, true); + ResourceInformation[] clusterRes = clusterResource.getResources(); + // We have to provide the calculateShares() method with somewhere to store + // the shares. We don't actually need these shares afterwards. + double[] numeratorShares = new double[clusterRes.length]; + double[] denominatorShares = new double[clusterRes.length]; + // We also have to provide a place for calculateShares() to store the max + // shares so that we can used them. + double[] max = new double[2]; + + calculateShares(clusterRes, numerator, denominator, numeratorShares, + denominatorShares, max); + + return (float) (max[0] / max[1]); } @Override