diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 9a8e2ec95df..86fcd7b5177 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -26,7 +26,6 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.Records; import java.util.Collections; @@ -258,18 +257,16 @@ public void setMemorySize(long memory) { * * @param resource name of the resource * @return the ResourceInformation object for the resource - * @throws YarnException if the resource can't be found + * @throws ResourceNotFoundException if the resource can't be found */ @Public @Evolving public ResourceInformation getResourceInformation(String resource) - throws YarnException { + throws ResourceNotFoundException { if (getResources().containsKey(resource)) { return getResources().get(resource); } - throw new YarnException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + throw new ResourceNotFoundException(resource); } /** @@ -278,17 +275,16 @@ public ResourceInformation getResourceInformation(String resource) * * @param resource name of the resource * @return the value for the resource - * @throws YarnException if the resource can't be found + * @throws ResourceNotFoundException if the resource can't be found */ @Public @Evolving - public Long getResourceValue(String resource) throws YarnException { + public Long getResourceValue(String resource) + throws ResourceNotFoundException { if (getResources().containsKey(resource)) { return getResources().get(resource).getValue(); } - throw new YarnException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + throw new ResourceNotFoundException(resource); } /** @@ -315,9 +311,7 @@ public void setResourceInformation(String resource, .copy(resourceInformation, getResources().get(resource)); return; } - throw new ResourceNotFoundException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + throw new ResourceNotFoundException(resource); } /** @@ -330,23 +324,21 @@ public void setResourceInformation(String resource, */ @Public @Evolving - public void setResourceValue(String resource, Long value) + public void setResourceValue(String resource, long value) throws ResourceNotFoundException { if (resource.equals(ResourceInformation.MEMORY_MB.getName())) { this.setMemorySize(value); return; } if (resource.equals(ResourceInformation.VCORES.getName())) { - this.setVirtualCores(value.intValue()); + this.setVirtualCores((int)value); return; } if (getResources().containsKey(resource)) { getResources().get(resource).setValue(value); return; } - throw new ResourceNotFoundException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + throw new ResourceNotFoundException(resource); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java index b5fece7dc8c..0b2220c01ff 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java @@ -28,18 +28,28 @@ @InterfaceAudience.Public @InterfaceStability.Unstable public class ResourceNotFoundException extends YarnRuntimeException { - private static final long serialVersionUID = 10081982L; + private static final String MESSAGE = "Requested unknown resource type: %s. " + + "The resource manager is in an inconsistent state and " + + "should be restarted. To minimize cluster down time enabling " + + "resource manager high avaliability is recommended."; + private final String resource; - public ResourceNotFoundException(String message) { - super(message); + public ResourceNotFoundException(String resource) { + super(String.format(MESSAGE, resource)); + this.resource = resource; } - public ResourceNotFoundException(Throwable cause) { - super(cause); + public ResourceNotFoundException(String resource, Throwable cause) { + super(String.format(MESSAGE, resource), cause); + this.resource = resource; } - public ResourceNotFoundException(String message, Throwable cause) { - super(message, cause); + /** + * Return the resource that was not found. + * @return the resource that was not found + */ + public String getResource() { + return resource; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 7bc7f5f8b7f..aa9d7ebc457 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -30,8 +30,8 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto; -import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.UnitsConversionUtil; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import java.util.HashMap; import java.util.Map; @@ -180,7 +180,7 @@ public void setResourceInformation(String resource, } @Override - public void setResourceValue(String resource, Long value) + public void setResourceValue(String resource, long value) throws ResourceNotFoundException { maybeInitBuilder(); initResources(); @@ -188,8 +188,7 @@ public void setResourceValue(String resource, Long value) throw new IllegalArgumentException("resource type object cannot be null"); } if (resources == null || (!resources.containsKey(resource))) { - throw new ResourceNotFoundException( - "Resource " + resource + " not found"); + throw new ResourceNotFoundException(resource); } resources.get(resource).setValue(value); } @@ -206,7 +205,8 @@ public ResourceInformation getResourceInformation(String resource) { if (this.resources.containsKey(resource)) { return this.resources.get(resource); } - throw new ResourceNotFoundException("Could not find entry for " + resource); + + throw new ResourceNotFoundException(resource); } @Override @@ -215,7 +215,8 @@ public Long getResourceValue(String resource) { if (this.resources.containsKey(resource)) { return this.resources.get(resource).getValue(); } - throw new ResourceNotFoundException("Could not find entry for " + resource); + + throw new ResourceNotFoundException(resource); } private void initResourcesMap() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 79bb03d113c..5e312defebb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -23,7 +23,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.UnitsConversionUtil; import java.util.Set; @@ -77,22 +76,18 @@ private int compare(Resource lhs, Resource rhs) { int ret = 0; for (String rName : resourceNames) { - try { - ResourceInformation lhsResourceInformation = - lhs.getResourceInformation(rName); - ResourceInformation rhsResourceInformation = - rhs.getResourceInformation(rName); - int diff = lhsResourceInformation.compareTo(rhsResourceInformation); - if (diff >= 1) { - lhsGreater = true; - } else if (diff <= -1) { - rhsGreater = true; - } - } catch (YarnException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + rName, ye); + ResourceInformation lhsResourceInformation = + lhs.getResourceInformation(rName); + ResourceInformation rhsResourceInformation = + rhs.getResourceInformation(rName); + int diff = lhsResourceInformation.compareTo(rhsResourceInformation); + if (diff >= 1) { + lhsGreater = true; + } else if (diff <= -1) { + rhsGreater = true; } } + if (lhsGreater && rhsGreater) { ret = 0; } else if (lhsGreater) { @@ -149,25 +144,21 @@ protected float getResourceAsValue(Resource clusterResource, float min = Float.MAX_VALUE; float max = 0.0f; for (String rName : resourceNames) { - try { - ResourceInformation clusterResourceResourceInformation = - clusterResource.getResourceInformation(rName); - ResourceInformation resourceInformation = - resource.getResourceInformation(rName); - long resourceValue = UnitsConversionUtil - .convert(resourceInformation.getUnits(), - clusterResourceResourceInformation.getUnits(), - resourceInformation.getValue()); - float tmp = - (float) resourceValue / (float) clusterResourceResourceInformation - .getValue(); - min = min < tmp ? min : tmp; - max = max > tmp ? max : tmp; - } catch (YarnException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); - } + ResourceInformation clusterResourceResourceInformation = + clusterResource.getResourceInformation(rName); + ResourceInformation resourceInformation = + resource.getResourceInformation(rName); + long resourceValue = UnitsConversionUtil + .convert(resourceInformation.getUnits(), + clusterResourceResourceInformation.getUnits(), + resourceInformation.getValue()); + float tmp = + (float) resourceValue / (float) clusterResourceResourceInformation + .getValue(); + min = min < tmp ? min : tmp; + max = max > tmp ? max : tmp; } + return (dominant) ? max : min; } @@ -175,24 +166,19 @@ protected float getResourceAsValue(Resource clusterResource, public long computeAvailableContainers(Resource available, Resource required) { long min = Long.MAX_VALUE; for (String resource : resourceNames) { - try { - ResourceInformation availableResource = - available.getResourceInformation(resource); - ResourceInformation requiredResource = - required.getResourceInformation(resource); - long requiredResourceValue = UnitsConversionUtil - .convert(requiredResource.getUnits(), availableResource.getUnits(), - requiredResource.getValue()); - if (requiredResourceValue != 0) { - long tmp = availableResource.getValue() / requiredResourceValue; - min = min < tmp ? min : tmp; - } - } catch (YarnException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); + ResourceInformation availableResource = + available.getResourceInformation(resource); + ResourceInformation requiredResource = + required.getResourceInformation(resource); + long requiredResourceValue = UnitsConversionUtil + .convert(requiredResource.getUnits(), availableResource.getUnits(), + requiredResource.getValue()); + if (requiredResourceValue != 0) { + long tmp = availableResource.getValue() / requiredResourceValue; + min = min < tmp ? min : tmp; } - } + return min > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) min; } @@ -207,15 +193,11 @@ public float divide(Resource clusterResource, @Override public boolean isInvalidDivisor(Resource r) { for (String resource : resourceNames) { - try { - if (r.getResourceValue(resource).equals(0L)) { - return true; - } - } catch (YarnException ye) { - throw new IllegalArgumentException( - "Error getting resource value for " + resource, ye); + if (r.getResourceValue(resource).equals(0L)) { + return true; } } + return false; } @@ -223,23 +205,19 @@ public boolean isInvalidDivisor(Resource r) { public float ratio(Resource a, Resource b) { float ratio = 0.0f; for (String resource : resourceNames) { - try { - ResourceInformation aResourceInformation = - a.getResourceInformation(resource); - ResourceInformation bResourceInformation = - b.getResourceInformation(resource); - long bResourceValue = UnitsConversionUtil - .convert(bResourceInformation.getUnits(), - aResourceInformation.getUnits(), - bResourceInformation.getValue()); - float tmp = - (float) aResourceInformation.getValue() / (float) bResourceValue; - ratio = ratio > tmp ? ratio : tmp; - } catch (YarnException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); - } + ResourceInformation aResourceInformation = + a.getResourceInformation(resource); + ResourceInformation bResourceInformation = + b.getResourceInformation(resource); + long bResourceValue = UnitsConversionUtil + .convert(bResourceInformation.getUnits(), + aResourceInformation.getUnits(), + bResourceInformation.getValue()); + float tmp = + (float) aResourceInformation.getValue() / (float) bResourceValue; + ratio = ratio > tmp ? ratio : tmp; } + return ratio; } @@ -251,16 +229,12 @@ public Resource divideAndCeil(Resource numerator, int denominator) { public Resource divideAndCeil(Resource numerator, long denominator) { Resource ret = Resource.newInstance(numerator); for (String resource : resourceNames) { - try { - ResourceInformation resourceInformation = - ret.getResourceInformation(resource); - resourceInformation.setValue( - divideAndCeil(resourceInformation.getValue(), denominator)); - } catch (YarnException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); - } + ResourceInformation resourceInformation = + ret.getResourceInformation(resource); + resourceInformation.setValue( + divideAndCeil(resourceInformation.getValue(), denominator)); } + return ret; } @@ -277,41 +251,38 @@ public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { - try { - ResourceInformation rResourceInformation = - r.getResourceInformation(resource); - ResourceInformation minimumResourceInformation = - minimumResource.getResourceInformation(resource); - ResourceInformation maximumResourceInformation = - maximumResource.getResourceInformation(resource); - ResourceInformation stepFactorResourceInformation = - stepFactor.getResourceInformation(resource); - ResourceInformation tmp = ret.getResourceInformation(resource); - - long rValue = rResourceInformation.getValue(); - long minimumValue = UnitsConversionUtil - .convert(minimumResourceInformation.getUnits(), - rResourceInformation.getUnits(), - minimumResourceInformation.getValue()); - long maximumValue = UnitsConversionUtil - .convert(maximumResourceInformation.getUnits(), - rResourceInformation.getUnits(), - maximumResourceInformation.getValue()); - long stepFactorValue = UnitsConversionUtil - .convert(stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); - long value = Math.max(rValue, minimumValue); - if (stepFactorValue != 0) { - value = roundUp(value, stepFactorValue); - } - tmp.setValue(Math.min(value, maximumValue)); - ret.setResourceInformation(resource, tmp); - } catch (YarnException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); + ResourceInformation rResourceInformation = + r.getResourceInformation(resource); + ResourceInformation minimumResourceInformation = + minimumResource.getResourceInformation(resource); + ResourceInformation maximumResourceInformation = + maximumResource.getResourceInformation(resource); + ResourceInformation stepFactorResourceInformation = + stepFactor.getResourceInformation(resource); + ResourceInformation tmp = ret.getResourceInformation(resource); + + long rValue = rResourceInformation.getValue(); + long minimumValue = UnitsConversionUtil + .convert(minimumResourceInformation.getUnits(), + rResourceInformation.getUnits(), + minimumResourceInformation.getValue()); + long maximumValue = UnitsConversionUtil + .convert(maximumResourceInformation.getUnits(), + rResourceInformation.getUnits(), + maximumResourceInformation.getValue()); + long stepFactorValue = UnitsConversionUtil + .convert(stepFactorResourceInformation.getUnits(), + rResourceInformation.getUnits(), + stepFactorResourceInformation.getValue()); + long value = Math.max(rValue, minimumValue); + if (stepFactorValue != 0) { + value = roundUp(value, stepFactorValue); } + tmp.setValue(Math.min(value, maximumValue)); + ret.setResourceInformation(resource, tmp); + tmp.setValue(Math.min(value, maximumValue)); } + return ret; } @@ -328,30 +299,26 @@ public Resource roundDown(Resource r, Resource stepFactor) { private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) { Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { - try { - ResourceInformation rResourceInformation = - r.getResourceInformation(resource); - ResourceInformation stepFactorResourceInformation = - stepFactor.getResourceInformation(resource); - - long rValue = rResourceInformation.getValue(); - long stepFactorValue = UnitsConversionUtil - .convert(stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); - long value = rValue; - if (stepFactorValue != 0) { - value = roundUp ? roundUp(rValue, stepFactorValue) : - roundDown(rValue, stepFactorValue); - } - ResourceInformation - .copy(rResourceInformation, ret.getResourceInformation(resource)); - ret.getResourceInformation(resource).setValue(value); - } catch (YarnException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); + ResourceInformation rResourceInformation = + r.getResourceInformation(resource); + ResourceInformation stepFactorResourceInformation = + stepFactor.getResourceInformation(resource); + + long rValue = rResourceInformation.getValue(); + long stepFactorValue = UnitsConversionUtil + .convert(stepFactorResourceInformation.getUnits(), + rResourceInformation.getUnits(), + stepFactorResourceInformation.getValue()); + long value = rValue; + if (stepFactorValue != 0) { + value = roundUp ? roundUp(rValue, stepFactorValue) : + roundDown(rValue, stepFactorValue); } + ResourceInformation + .copy(rResourceInformation, ret.getResourceInformation(resource)); + ret.getResourceInformation(resource).setValue(value); } + return ret; } @@ -371,55 +338,48 @@ private Resource multiplyAndNormalize(Resource r, double by, Resource stepFactor, boolean roundUp) { Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { - try { - ResourceInformation rResourceInformation = - r.getResourceInformation(resource); - ResourceInformation stepFactorResourceInformation = - stepFactor.getResourceInformation(resource); - ResourceInformation tmp = ret.getResourceInformation(resource); - - Long rValue = rResourceInformation.getValue(); - Long stepFactorValue = UnitsConversionUtil - .convert(stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); - Long value; - if (stepFactorValue != 0) { - value = roundUp ? - roundUp((long) Math.ceil(rValue * by), stepFactorValue) : - roundDown((long) (rValue * by), stepFactorValue); - } else { - value = - roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by); - } - tmp.setValue(value); - } catch (YarnException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); + ResourceInformation rResourceInformation = + r.getResourceInformation(resource); + ResourceInformation stepFactorResourceInformation = + stepFactor.getResourceInformation(resource); + ResourceInformation tmp = ret.getResourceInformation(resource); + + Long rValue = rResourceInformation.getValue(); + Long stepFactorValue = UnitsConversionUtil + .convert(stepFactorResourceInformation.getUnits(), + rResourceInformation.getUnits(), + stepFactorResourceInformation.getValue()); + Long value; + if (stepFactorValue != 0) { + value = roundUp ? + roundUp((long) Math.ceil(rValue * by), stepFactorValue) : + roundDown((long) (rValue * by), stepFactorValue); + } else { + value = + roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by); } + tmp.setValue(value); } + return ret; } @Override public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) { for (String resource : resourceNames) { - try { - ResourceInformation sResourceInformation = - smaller.getResourceInformation(resource); - ResourceInformation bResourceInformation = - bigger.getResourceInformation(resource); - long sResourceValue = UnitsConversionUtil - .convert(sResourceInformation.getUnits(), - bResourceInformation.getUnits(), - sResourceInformation.getValue()); - if(sResourceValue > bResourceInformation.getValue()) { - return false; - } - } catch (YarnException ye) { + ResourceInformation sResourceInformation = + smaller.getResourceInformation(resource); + ResourceInformation bResourceInformation = + bigger.getResourceInformation(resource); + long sResourceValue = UnitsConversionUtil + .convert(sResourceInformation.getUnits(), + bResourceInformation.getUnits(), + sResourceInformation.getValue()); + if(sResourceValue > bResourceInformation.getValue()) { return false; } } + return true; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 86cf872b160..7887d694aa7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -287,7 +287,7 @@ static void initializeResourcesMap(Configuration conf, lock = new Object(); } catch (FileNotFoundException fe) { LOG.info("Unable to find '" + resourceFile - + "'. Falling back to memory and vcores as resources", fe); + + "'. Falling back to memory and vcores as resources"); initializeResourcesMap(conf, resources); lock = new Object(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index d143e93e88f..a6a27a166f2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -23,8 +23,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.UnitsConversionUtil; import java.util.Collections; @@ -102,22 +100,25 @@ public void setVirtualCores(int virtualCores) { @Override public ResourceInformation getResourceInformation(String resource) - throws YarnException { + throws ResourceNotFoundException { if (resources.containsKey(resource)) { ResourceInformation value = this.resources.get(resource); ResourceInformation ret = ResourceInformation.newInstance(value); ret.setValue(resourceValue); return ret; } - throw new YarnException("" + resource + " not found"); + + throw new ResourceNotFoundException(resource); } @Override - public Long getResourceValue(String resource) throws YarnException { + public Long getResourceValue(String resource) + throws ResourceNotFoundException { if (resources.containsKey(resource)) { return resourceValue; } - throw new YarnException("" + resource + " not found"); + + throw new ResourceNotFoundException(resource); } @Override @@ -128,7 +129,7 @@ public void setResourceInformation(String resource, } @Override - public void setResourceValue(String resource, Long value) + public void setResourceValue(String resource, long value) throws ResourceNotFoundException { throw new RuntimeException(name + " cannot be modified!"); } @@ -207,7 +208,7 @@ public static Resource addTo(Resource lhs, Resource rhs) { .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { continue; } } @@ -229,7 +230,7 @@ public static Resource subtractFrom(Resource lhs, Resource rhs) { .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); lhs.setResourceValue(name, lhsValue.getValue() - convertedRhs); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { continue; } } @@ -292,7 +293,7 @@ public static Resource multiplyAndAddTo( .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()) * by); lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { continue; } } @@ -428,7 +429,7 @@ public static boolean fitsIn(Resource smaller, Resource bigger) { if(lhsValue.getValue() > convertedRhs) { return false; } - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { return false; } } @@ -454,7 +455,7 @@ public static Resource componentwiseMin(Resource lhs, Resource rhs) { ResourceInformation outInfo = lhsValue.getValue() < convertedRhs ? lhsValue : rhsValue; ret.setResourceInformation(name, outInfo); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { continue; } } @@ -475,7 +476,7 @@ public static Resource componentwiseMax(Resource lhs, Resource rhs) { ResourceInformation outInfo = lhsValue.getValue() > convertedRhs ? lhsValue : rhsValue; ret.setResourceInformation(name, outInfo); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { continue; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceWeights.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceWeights.java index 4c62318e6f4..d4da603bb61 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceWeights.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceWeights.java @@ -18,42 +18,91 @@ package org.apache.hadoop.yarn.server.resourcemanager.resource; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.ListIterator; +import java.util.stream.Collectors; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; @Private @Evolving public class ResourceWeights { public static final ResourceWeights NEUTRAL = new ResourceWeights(1.0f); + private final List resources; - private float[] weights = new float[ResourceType.values().length]; + public ResourceWeights(float weight) { + this(); + setWeight(weight); + } - public ResourceWeights(float memoryWeight, float cpuWeight) { - weights[ResourceType.MEMORY.ordinal()] = memoryWeight; - weights[ResourceType.CPU.ordinal()] = cpuWeight; + public ResourceWeights() { + Collection infos = + ResourceUtils.getResourceTypes().values(); + + resources = new ArrayList(infos.stream().map(info -> + new ResourceWeight(info.getName(), 0.0f)).collect(Collectors.toList())); } - public ResourceWeights(float weight) { - setWeight(weight); + public final void setWeight(float weight) { + resources.forEach(resource -> resource.weight = weight); } - public ResourceWeights() { } + public void setWeight(String resource, float weight) { + int index = indexOfResouce(resource); - public void setWeight(float weight) { - for (int i = 0; i < weights.length; i++) { - weights[i] = weight; + if (index >= 0) { + resources.get(index).weight = weight; + } else { + throw new ResourceNotFoundException(resource); } } - public void setWeight(ResourceType resourceType, float weight) { - weights[resourceType.ordinal()] = weight; + private int indexOfResouce(String resource) { + // While searching a list isn't very efficient, the list is always going + // to be short, and the alternative is worse. + ListIterator itr = resources.listIterator(); + int index = -1; + + while (itr.hasNext()) { + if (itr.next().resource.equals(resource)) { + index = itr.previousIndex(); + break; + } + } + + return index; + } + + public float getWeight(String resource) { + int index = indexOfResouce(resource); + float weight = 0.0f; + + if (index >= 0) { + weight = resources.get(index).weight; + } else { + throw new ResourceNotFoundException(resource); + } + + return weight; } - - public float getWeight(ResourceType resourceType) { - return weights[resourceType.ordinal()]; + + /** + * Returns a list of resource names sorted in descending order by weights. + * @return a sorted list of resource names + */ + public List getResourcesSortedByWeight() { + Collections.sort(resources); + + return resources.stream().map(e -> e.resource).collect(Collectors.toList()); } - + public String toString() { StringBuffer sb = new StringBuffer(); sb.append("<"); @@ -61,11 +110,30 @@ public String toString() { if (i != 0) { sb.append(", "); } - ResourceType resourceType = ResourceType.values()[i]; - sb.append(StringUtils.toLowerCase(resourceType.name())); + String resourceType = resources.get(i).resource; + sb.append(StringUtils.toLowerCase(resourceType)); sb.append(StringUtils.format(" weight=%.1f", getWeight(resourceType))); } sb.append(">"); return sb.toString(); } + + /** + * A simple holder class for sorting by weight. + */ + private static class ResourceWeight implements Comparable { + private final String resource; + private float weight; + + private ResourceWeight(String resource, float weight) { + this.resource = resource; + this.weight = weight; + } + + @Override + public int compareTo(ResourceWeight o) { + // Natural sort order is descending. + return (int)Math.signum(o.weight - weight); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java index 440c73cefdd..0b5b6a4a530 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java @@ -47,7 +47,7 @@ */ public static void computeShares( Collection schedulables, Resource totalResources, - ResourceType type) { + String type) { computeSharesInternal(schedulables, totalResources, type, false); } @@ -62,7 +62,7 @@ public static void computeShares( */ public static void computeSteadyShares( Collection queues, Resource totalResources, - ResourceType type) { + String type) { computeSharesInternal(queues, totalResources, type, true); } @@ -110,9 +110,9 @@ public static void computeSteadyShares( */ private static void computeSharesInternal( Collection allSchedulables, - Resource totalResources, ResourceType type, boolean isSteadyShare) { + Resource totalResources, String type, boolean isSteadyShare) { - Collection schedulables = new ArrayList(); + Collection schedulables = new ArrayList<>(); int takenResources = handleFixedFairShares( allSchedulables, schedulables, isSteadyShare, type); @@ -124,7 +124,7 @@ private static void computeSharesInternal( // have met all Schedulables' max shares. int totalMaxShare = 0; for (Schedulable sched : schedulables) { - long maxShare = getResourceValue(sched.getMaxShare(), type); + long maxShare = sched.getMaxShare().getResourceValue(type); totalMaxShare = (int) Math.min(maxShare + (long)totalMaxShare, Integer.MAX_VALUE); if (totalMaxShare == Integer.MAX_VALUE) { @@ -132,7 +132,7 @@ private static void computeSharesInternal( } } - long totalResource = Math.max((getResourceValue(totalResources, type) - + long totalResource = Math.max((totalResources.getResourceValue(type) - takenResources), 0); totalResource = Math.min(totalMaxShare, totalResource); @@ -159,13 +159,15 @@ private static void computeSharesInternal( } // Set the fair shares based on the value of R we've converged to for (Schedulable sched : schedulables) { + Resource target; + if (isSteadyShare) { - setResourceValue(computeShare(sched, right, type), - ((FSQueue) sched).getSteadyFairShare(), type); + target = ((FSQueue) sched).getSteadyFairShare(); } else { - setResourceValue( - computeShare(sched, right, type), sched.getFairShare(), type); + target = sched.getFairShare(); } + + target.setResourceValue(type, computeShare(sched, right, type)); } } @@ -174,7 +176,7 @@ private static void computeSharesInternal( * w2rRatio, for use in the computeFairShares algorithm as described in # */ private static int resourceUsedWithWeightToResourceRatio(double w2rRatio, - Collection schedulables, ResourceType type) { + Collection schedulables, String type) { int resourcesTaken = 0; for (Schedulable sched : schedulables) { int share = computeShare(sched, w2rRatio, type); @@ -188,10 +190,10 @@ private static int resourceUsedWithWeightToResourceRatio(double w2rRatio, * weight-to-resource ratio w2rRatio. */ private static int computeShare(Schedulable sched, double w2rRatio, - ResourceType type) { + String type) { double share = sched.getWeights().getWeight(type) * w2rRatio; - share = Math.max(share, getResourceValue(sched.getMinShare(), type)); - share = Math.min(share, getResourceValue(sched.getMaxShare(), type)); + share = Math.max(share, sched.getMinShare().getResourceValue(type)); + share = Math.min(share, sched.getMaxShare().getResourceValue(type)); return (int) share; } @@ -203,7 +205,7 @@ private static int computeShare(Schedulable sched, double w2rRatio, private static int handleFixedFairShares( Collection schedulables, Collection nonFixedSchedulables, - boolean isSteadyShare, ResourceType type) { + boolean isSteadyShare, String type) { int totalResource = 0; for (Schedulable sched : schedulables) { @@ -211,11 +213,15 @@ private static int handleFixedFairShares( if (fixedShare < 0) { nonFixedSchedulables.add(sched); } else { - setResourceValue(fixedShare, - isSteadyShare - ? ((FSQueue)sched).getSteadyFairShare() - : sched.getFairShare(), - type); + Resource target; + + if (isSteadyShare) { + target = ((FSQueue)sched).getSteadyFairShare(); + } else { + target = sched.getFairShare(); + } + + target.setResourceValue(type, fixedShare); totalResource = (int) Math.min((long)totalResource + (long)fixedShare, Integer.MAX_VALUE); } @@ -230,10 +236,10 @@ private static int handleFixedFairShares( * or the Schedulable is not active for instantaneous fairshare. */ private static long getFairShareIfFixed(Schedulable sched, - boolean isSteadyShare, ResourceType type) { + boolean isSteadyShare, String type) { // Check if maxShare is 0 - if (getResourceValue(sched.getMaxShare(), type) <= 0) { + if (sched.getMaxShare().getResourceValue(type) <= 0) { return 0; } @@ -245,7 +251,7 @@ private static long getFairShareIfFixed(Schedulable sched, // Check if weight is 0 if (sched.getWeights().getWeight(type) <= 0) { - long minShare = getResourceValue(sched.getMinShare(), type); + long minShare = sched.getMinShare().getResourceValue(type); return (minShare <= 0) ? 0 : minShare; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java index 193ed4dc36c..bb22143bfd7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java @@ -20,11 +20,13 @@ import java.util.Collection; import java.util.Comparator; +import java.util.List; +import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; @@ -34,8 +36,8 @@ import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; -import static org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType.*; /** * Makes scheduling decisions by trying to equalize dominant resource usage. @@ -52,6 +54,8 @@ new DominantResourceFairnessComparator(); private static final DominantResourceCalculator CALCULATOR = new DominantResourceCalculator(); + private static final Map RESOURCES = + ResourceUtils.getResourceTypes(); @Override public String getName() { @@ -71,17 +75,17 @@ public ResourceCalculator getResourceCalculator() { @Override public void computeShares(Collection schedulables, Resource totalResources) { - for (ResourceType type : ResourceType.values()) { - ComputeFairShares.computeShares(schedulables, totalResources, type); - } + RESOURCES.values().forEach(info -> + ComputeFairShares.computeShares(schedulables, totalResources, + info.getName())); } @Override public void computeSteadyShares(Collection queues, Resource totalResources) { - for (ResourceType type : ResourceType.values()) { - ComputeFairShares.computeSteadyShares(queues, totalResources, type); - } + RESOURCES.values().forEach(info -> + ComputeFairShares.computeSteadyShares(queues, totalResources, + info.getName())); } @Override @@ -110,8 +114,6 @@ public void initialize(FSContext fsContext) { } public static class DominantResourceFairnessComparator implements Comparator { - private static final int NUM_RESOURCES = ResourceType.values().length; - private FSContext fsContext; public void setFSContext(FSContext fsContext) { @@ -124,27 +126,34 @@ public int compare(Schedulable s1, Schedulable s2) { ResourceWeights sharesOfCluster2 = new ResourceWeights(); ResourceWeights sharesOfMinShare1 = new ResourceWeights(); ResourceWeights sharesOfMinShare2 = new ResourceWeights(); - ResourceType[] resourceOrder1 = new ResourceType[NUM_RESOURCES]; - ResourceType[] resourceOrder2 = new ResourceType[NUM_RESOURCES]; Resource clusterCapacity = fsContext.getClusterResource(); // Calculate shares of the cluster for each resource both schedulables. calculateShares(s1.getResourceUsage(), - clusterCapacity, sharesOfCluster1, resourceOrder1, s1.getWeights()); + clusterCapacity, sharesOfCluster1, s1.getWeights()); calculateShares(s1.getResourceUsage(), - s1.getMinShare(), sharesOfMinShare1, null, ResourceWeights.NEUTRAL); + s1.getMinShare(), sharesOfMinShare1, ResourceWeights.NEUTRAL); calculateShares(s2.getResourceUsage(), - clusterCapacity, sharesOfCluster2, resourceOrder2, s2.getWeights()); + clusterCapacity, sharesOfCluster2, s2.getWeights()); calculateShares(s2.getResourceUsage(), - s2.getMinShare(), sharesOfMinShare2, null, ResourceWeights.NEUTRAL); - - // A queue is needy for its min share if its dominant resource + s2.getMinShare(), sharesOfMinShare2, ResourceWeights.NEUTRAL); + + // sort order vector by resource share + List resourceOrder1 = + sharesOfCluster1.getResourcesSortedByWeight(); + List resourceOrder2 = + sharesOfCluster2.getResourcesSortedByWeight(); + + // A queue is needy for its min share of its dominant resource // (with respect to the cluster capacity) is below its configured min share // for that resource - boolean s1Needy = sharesOfMinShare1.getWeight(resourceOrder1[0]) < 1.0f; - boolean s2Needy = sharesOfMinShare2.getWeight(resourceOrder2[0]) < 1.0f; + boolean s1Needy = + sharesOfMinShare1.getWeight(resourceOrder1.get(0)) < 1.0f; + boolean s2Needy = + sharesOfMinShare2.getWeight(resourceOrder2.get(0)) < 1.0f; int res = 0; + if (!s2Needy && !s1Needy) { res = compareShares(sharesOfCluster1, sharesOfCluster2, resourceOrder1, resourceOrder2); @@ -156,6 +165,7 @@ public int compare(Schedulable s1, Schedulable s2) { res = compareShares(sharesOfMinShare1, sharesOfMinShare2, resourceOrder1, resourceOrder2); } + if (res == 0) { // Apps are tied in fairness ratio. Break the tie by submit time and job // name to get a deterministic ordering, which is useful for unit tests. @@ -175,28 +185,17 @@ public int compare(Schedulable s1, Schedulable s2) { * shares will be [.1, .5] and resourceOrder will be [CPU, MEMORY]. */ void calculateShares(Resource resource, Resource pool, - ResourceWeights shares, ResourceType[] resourceOrder, ResourceWeights weights) { - shares.setWeight(MEMORY, (float)resource.getMemorySize() / - (pool.getMemorySize() * weights.getWeight(MEMORY))); - shares.setWeight(CPU, (float)resource.getVirtualCores() / - (pool.getVirtualCores() * weights.getWeight(CPU))); - // sort order vector by resource share - if (resourceOrder != null) { - if (shares.getWeight(MEMORY) > shares.getWeight(CPU)) { - resourceOrder[0] = MEMORY; - resourceOrder[1] = CPU; - } else { - resourceOrder[0] = CPU; - resourceOrder[1] = MEMORY; - } - } + ResourceWeights shares, ResourceWeights weights) { + RESOURCES.keySet().forEach(info -> + shares.setWeight(info, resource.getResourceValue(info) / + (pool.getResourceValue(info) * weights.getWeight(info)))); } private int compareShares(ResourceWeights shares1, ResourceWeights shares2, - ResourceType[] resourceOrder1, ResourceType[] resourceOrder2) { - for (int i = 0; i < resourceOrder1.length; i++) { - int ret = (int)Math.signum(shares1.getWeight(resourceOrder1[i]) - - shares2.getWeight(resourceOrder2[i])); + List resourceOrder1, List resourceOrder2) { + for (int i = 0; i < resourceOrder1.size(); i++) { + int ret = (int)Math.signum(shares1.getWeight(resourceOrder1.get(i)) + - shares2.getWeight(resourceOrder2.get(i))); if (ret != 0) { return ret; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java index c3ec47a1eba..89ebd6d9f47 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.api.records.ResourceInformation; /** * Makes scheduling decisions by trying to equalize shares of memory. @@ -43,6 +44,7 @@ @Unstable public class FairSharePolicy extends SchedulingPolicy { private static final Log LOG = LogFactory.getLog(FairSharePolicy.class); + private static String MEMORY = ResourceInformation.MEMORY_MB.getName(); @VisibleForTesting public static final String NAME = "fair"; private static final DefaultResourceCalculator RESOURCE_CALCULATOR = @@ -98,8 +100,8 @@ public int compare(Schedulable s1, Schedulable s2) { minShareRatio2 = (double) resourceUsage2.getMemorySize() / Resources.max(RESOURCE_CALCULATOR, null, minShare2, ONE).getMemorySize(); - weight1 = s1.getWeights().getWeight(ResourceType.MEMORY); - weight2 = s2.getWeights().getWeight(ResourceType.MEMORY); + weight1 = s1.getWeights().getWeight(MEMORY); + weight2 = s2.getWeights().getWeight(MEMORY); if (weight1 > 0.0 && weight2 > 0.0) { useToWeightRatio1 = resourceUsage1.getMemorySize() / weight1; useToWeightRatio2 = resourceUsage2.getMemorySize() / weight2; @@ -163,14 +165,13 @@ public Resource getHeadroom(Resource queueFairShare, @Override public void computeShares(Collection schedulables, Resource totalResources) { - ComputeFairShares.computeShares(schedulables, totalResources, ResourceType.MEMORY); + ComputeFairShares.computeShares(schedulables, totalResources, MEMORY); } @Override public void computeSteadyShares(Collection queues, Resource totalResources) { - ComputeFairShares.computeSteadyShares(queues, totalResources, - ResourceType.MEMORY); + ComputeFairShares.computeSteadyShares(queues, totalResources, MEMORY); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java index 9561234d633..bedc7627ed3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java @@ -34,11 +34,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.ReservationAgent; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -137,7 +137,8 @@ protected void checkDefaultQueueBeforePlanFollowerRun() { } @Override protected void verifyCapacity(Queue defQ) { - assertTrue(((FSQueue) defQ).getWeights().getWeight(ResourceType.MEMORY) > 0.9); + assertTrue(((FSQueue) defQ).getWeights() + .getWeight(ResourceInformation.MEMORY_MB.getName()) > 0.9); } @Override @@ -174,7 +175,8 @@ protected void assertReservationQueueExists(ReservationId r, assertNotNull(q); // For now we are setting both to same weight Assert.assertEquals(expectedCapacity, - q.getWeights().getWeight(ResourceType.MEMORY), 0.01); + q.getWeights().getWeight(ResourceInformation.MEMORY_MB.getName()), + 0.01); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceWeights.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceWeights.java index f420b9ecd22..059c71134d3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceWeights.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceWeights.java @@ -17,39 +17,31 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.junit.Assert; import org.junit.Test; public class TestResourceWeights { - @Test(timeout=3000) + @Test public void testWeights() { ResourceWeights rw1 = new ResourceWeights(); Assert.assertEquals("Default CPU weight should be 0.0f.", 0.0f, - rw1.getWeight(ResourceType.CPU), 0.00001f); + rw1.getWeight(ResourceInformation.VCORES.getName()), 0.00001f); Assert.assertEquals("Default memory weight should be 0.0f", 0.0f, - rw1.getWeight(ResourceType.MEMORY), 0.00001f); + rw1.getWeight(ResourceInformation.MEMORY_MB.getName()), 0.00001f); ResourceWeights rw2 = new ResourceWeights(2.0f); Assert.assertEquals("The CPU weight should be 2.0f.", 2.0f, - rw2.getWeight(ResourceType.CPU), 0.00001f); + rw2.getWeight(ResourceInformation.VCORES.getName()), 0.00001f); Assert.assertEquals("The memory weight should be 2.0f", 2.0f, - rw2.getWeight(ResourceType.MEMORY), 0.00001f); + rw2.getWeight(ResourceInformation.MEMORY_MB.getName()), 0.00001f); - // set each individually - ResourceWeights rw3 = new ResourceWeights(1.5f, 2.0f); - Assert.assertEquals("The CPU weight should be 2.0f", 2.0f, - rw3.getWeight(ResourceType.CPU), 0.00001f); - Assert.assertEquals("The memory weight should be 1.5f", 1.5f, - rw3.getWeight(ResourceType.MEMORY), 0.00001f); - - // reset weights - rw3.setWeight(ResourceType.CPU, 2.5f); - Assert.assertEquals("The CPU weight should be set to 2.5f.", 2.5f, - rw3.getWeight(ResourceType.CPU), 0.00001f); - rw3.setWeight(ResourceType.MEMORY, 4.0f); - Assert.assertEquals("The memory weight should be set to 4.0f.", 4.0f, - rw3.getWeight(ResourceType.MEMORY), 0.00001f); + ResourceWeights rwn = ResourceWeights.NEUTRAL; + Assert.assertEquals("Default CPU weight should be 1.0f.", 1.0f, + rwn.getWeight(ResourceInformation.VCORES.getName()), 1.00001f); + Assert.assertEquals("Default memory weight should be 1.0f", 1.0f, + rwn.getWeight(ResourceInformation.MEMORY_MB.getName()), 1.00001f); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java index 4f3ccb2acd4..bba09f1e080 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java @@ -20,11 +20,11 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.junit.Assert; import org.apache.hadoop.yarn.util.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.ComputeFairShares; import org.junit.Before; @@ -52,7 +52,7 @@ public void testEqualSharing() { scheds.add(new FakeSchedulable()); scheds.add(new FakeSchedulable()); ComputeFairShares.computeShares(scheds, - Resources.createResource(40), ResourceType.MEMORY); + Resources.createResource(40), ResourceInformation.MEMORY_MB.getName()); verifyMemoryShares(10, 10, 10, 10); } @@ -70,7 +70,7 @@ public void testLowMaxShares() { scheds.add(new FakeSchedulable(0, 11)); scheds.add(new FakeSchedulable(0, 3)); ComputeFairShares.computeShares(scheds, - Resources.createResource(40), ResourceType.MEMORY); + Resources.createResource(40), ResourceInformation.MEMORY_MB.getName()); verifyMemoryShares(13, 13, 11, 3); } @@ -90,7 +90,7 @@ public void testMinShares() { scheds.add(new FakeSchedulable(0)); scheds.add(new FakeSchedulable(2)); ComputeFairShares.computeShares(scheds, - Resources.createResource(40), ResourceType.MEMORY); + Resources.createResource(40), ResourceInformation.MEMORY_MB.getName()); verifyMemoryShares(20, 18, 0, 2); } @@ -105,7 +105,7 @@ public void testWeightedSharing() { scheds.add(new FakeSchedulable(0, 1.0)); scheds.add(new FakeSchedulable(0, 0.5)); ComputeFairShares.computeShares(scheds, - Resources.createResource(45), ResourceType.MEMORY); + Resources.createResource(45), ResourceInformation.MEMORY_MB.getName()); verifyMemoryShares(20, 10, 10, 5); } @@ -123,7 +123,7 @@ public void testWeightedSharingWithMaxShares() { scheds.add(new FakeSchedulable(0, 30, 1.0)); scheds.add(new FakeSchedulable(0, 20, 0.5)); ComputeFairShares.computeShares(scheds, - Resources.createResource(45), ResourceType.MEMORY); + Resources.createResource(45), ResourceInformation.MEMORY_MB.getName()); verifyMemoryShares(10, 11, 16, 8); } @@ -142,7 +142,7 @@ public void testWeightedSharingWithMinShares() { scheds.add(new FakeSchedulable(5, 1.0)); scheds.add(new FakeSchedulable(15, 0.5)); ComputeFairShares.computeShares(scheds, - Resources.createResource(45), ResourceType.MEMORY); + Resources.createResource(45), ResourceInformation.MEMORY_MB.getName()); verifyMemoryShares(20, 5, 5, 15); } @@ -158,7 +158,8 @@ public void testLargeShares() { scheds.add(new FakeSchedulable()); scheds.add(new FakeSchedulable()); ComputeFairShares.computeShares(scheds, - Resources.createResource(40 * million), ResourceType.MEMORY); + Resources.createResource(40 * million), + ResourceInformation.MEMORY_MB.getName()); verifyMemoryShares(10 * million, 10 * million, 10 * million, 10 * million); } @@ -168,7 +169,7 @@ public void testLargeShares() { @Test public void testEmptyList() { ComputeFairShares.computeShares(scheds, - Resources.createResource(40), ResourceType.MEMORY); + Resources.createResource(40), ResourceInformation.MEMORY_MB.getName()); verifyMemoryShares(); } @@ -186,7 +187,7 @@ public void testCPU() { scheds.add(new FakeSchedulable(Resources.createResource(0, 15), new ResourceWeights(0.5f))); ComputeFairShares.computeShares(scheds, - Resources.createResource(0, 45), ResourceType.CPU); + Resources.createResource(0, 45), ResourceInformation.VCORES.getName()); verifyCPUShares(20, 5, 5, 15); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 63dcade4c46..b6e61cf478c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -79,7 +79,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -123,6 +122,7 @@ import org.xml.sax.SAXException; import com.google.common.collect.Sets; +import org.apache.hadoop.yarn.api.records.ResourceInformation; @SuppressWarnings("unchecked") public class TestFairScheduler extends FairSchedulerTestBase { @@ -1984,7 +1984,8 @@ public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception { // assert that the steady fair share is 1/4th node1's capacity assertEquals(capacity / 4, leaf.getSteadyFairShare().getMemorySize()); // assert weights are equal for both the user queues - assertEquals(1.0, leaf.getWeights().getWeight(ResourceType.MEMORY), 0); + assertEquals(1.0, leaf.getWeights() + .getWeight(ResourceInformation.MEMORY_MB.getName()), 0); } } } @@ -5277,7 +5278,7 @@ public void testDumpState() throws IOException { child1.updateDemand(); String childQueueString = "{Name: root.parent.child1," - + " Weight: ," + + " Weight: ," + " Policy: fair," + " FairShare: ," + " SteadyFairShare: ," @@ -5294,14 +5295,15 @@ public void testDumpState() throws IOException { + " LastTimeAtMinShare: " + clock.getTime() + "}"; - assertTrue(child1.dumpState().equals(childQueueString)); + assertEquals("Unexpected state dump string", + childQueueString, child1.dumpState()); FSParentQueue parent = scheduler.getQueueManager().getParentQueue("parent", false); parent.setMaxShare(resource); parent.updateDemand(); String parentQueueString = "{Name: root.parent," - + " Weight: ," + + " Weight: ," + " Policy: fair," + " FairShare: ," + " SteadyFairShare: ," @@ -5312,7 +5314,7 @@ public void testDumpState() throws IOException { + " MaxAMShare: 0.5," + " Runnable: 0}"; - assertTrue(parent.dumpState().equals( - parentQueueString + ", " + childQueueString)); + assertEquals("Unexpected state dump string", + parentQueueString + ", " + childQueueString, parent.dumpState()); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java index 3719e2aee08..9008a902104 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java @@ -25,7 +25,7 @@ import java.util.Comparator; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FakeSchedulable; @@ -132,21 +132,41 @@ public void testEvenWeightsDifferentDominantResource() { @Test public void testUnevenWeightsSameDominantResource() { + ResourceWeights weights = new ResourceWeights(); + + weights.setWeight(ResourceInformation.MEMORY_MB.getName(), 2.0f); + weights.setWeight(ResourceInformation.VCORES.getName(), 1.0f); + assertTrue(createComparator(8000, 8).compare( - createSchedulable(3000, 1, new ResourceWeights(2.0f, 1.0f)), + createSchedulable(3000, 1, weights), createSchedulable(2000, 1)) < 0); + + weights.setWeight(ResourceInformation.MEMORY_MB.getName(), 1.0f); + weights.setWeight(ResourceInformation.VCORES.getName(), 2.0f); + assertTrue(createComparator(8000, 8).compare( - createSchedulable(1000, 3, new ResourceWeights(1.0f, 2.0f)), + createSchedulable(1000, 3, weights), createSchedulable(1000, 2)) < 0); } @Test public void testUnevenWeightsDifferentDominantResource() { - assertTrue(createComparator(8000, 8).compare( - createSchedulable(1000, 3, new ResourceWeights(1.0f, 2.0f)), + ResourceWeights weights = new ResourceWeights(); + + weights.setWeight(ResourceInformation.MEMORY_MB.getName(), 1.0f); + weights.setWeight(ResourceInformation.VCORES.getName(), 2.0f); + + assertTrue("Resource comparison resulted in wrong order", + createComparator(8000, 8).compare( + createSchedulable(1000, 3, weights), createSchedulable(2000, 1)) < 0); - assertTrue(createComparator(8000, 8).compare( - createSchedulable(3000, 1, new ResourceWeights(2.0f, 1.0f)), + + weights.setWeight(ResourceInformation.MEMORY_MB.getName(), 2.0f); + weights.setWeight(ResourceInformation.VCORES.getName(), 1.0f); + + assertTrue("Resource comparison resulted in wrong order", + createComparator(8000, 8).compare( + createSchedulable(3000, 1, weights), createSchedulable(1000, 2)) < 0); } @@ -154,17 +174,14 @@ public void testUnevenWeightsDifferentDominantResource() { public void testCalculateShares() { Resource used = Resources.createResource(10, 5); Resource capacity = Resources.createResource(100, 10); - ResourceType[] resourceOrder = new ResourceType[2]; ResourceWeights shares = new ResourceWeights(); DominantResourceFairnessPolicy.DominantResourceFairnessComparator comparator = new DominantResourceFairnessPolicy.DominantResourceFairnessComparator(); - comparator.calculateShares(used, capacity, shares, resourceOrder, + comparator.calculateShares(used, capacity, shares, ResourceWeights.NEUTRAL); - assertEquals(.1, shares.getWeight(ResourceType.MEMORY), .00001); - assertEquals(.5, shares.getWeight(ResourceType.CPU), .00001); - assertEquals(ResourceType.CPU, resourceOrder[0]); - assertEquals(ResourceType.MEMORY, resourceOrder[1]); + assertEquals(.1, shares.getWeight(ResourceInformation.MEMORY_MB.getName()), .00001); + assertEquals(.5, shares.getWeight(ResourceInformation.VCORES.getName()), .00001); } @Test