diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index a485a57da3b..d5177ffcfbb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -209,19 +209,16 @@ public void setMemorySize(long memory) { * * @param resource name of the resource * @return the ResourceInformation object for the resource - * @throws ResourceNotFoundException if the resource can't be found */ @Public @Evolving - public ResourceInformation getResourceInformation(String resource) - throws ResourceNotFoundException { + public ResourceInformation getResourceInformation(String resource) { Integer index = ResourceUtils.getResourceTypeIndex().get(resource); ResourceInformation[] resources = getResources(); if (index != null) { return resources[index]; } - throw new ResourceNotFoundException("Unknown resource '" + resource - + "'. Known resources are " + Arrays.toString(resources)); + throw new ResourceNotFoundException(resource); } /** @@ -230,12 +227,10 @@ public ResourceInformation getResourceInformation(String resource) * * @param resource name of the resource * @return the value for the resource - * @throws ResourceNotFoundException if the resource can't be found */ @Public @Evolving - public long getResourceValue(String resource) - throws ResourceNotFoundException { + public long getResourceValue(String resource) { return getResourceInformation(resource).getValue(); } @@ -244,13 +239,11 @@ public long getResourceValue(String resource) * * @param resource the resource for which the ResourceInformation is provided * @param resourceInformation ResourceInformation object - * @throws ResourceNotFoundException if the resource is not found */ @Public @Evolving public void setResourceInformation(String resource, - ResourceInformation resourceInformation) - throws ResourceNotFoundException { + ResourceInformation resourceInformation) { if (resource.equals(MEMORY)) { this.setMemorySize(resourceInformation.getValue()); return; @@ -269,12 +262,10 @@ public void setResourceInformation(String resource, * * @param resource the resource for which the value is provided. * @param value the value to set - * @throws ResourceNotFoundException if the resource is not found */ @Public @Evolving - public void setResourceValue(String resource, Long value) - throws ResourceNotFoundException { + public void setResourceValue(String resource, Long value) { if (resource.equals(MEMORY)) { this.setMemorySize(value); return; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java index b5fece7dc8c..ea8b946a973 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java @@ -28,18 +28,28 @@ @InterfaceAudience.Public @InterfaceStability.Unstable public class ResourceNotFoundException extends YarnRuntimeException { - private static final long serialVersionUID = 10081982L; + private static final String MESSAGE = "Requested unknown resource type: %s. " + + "The resource manager is in an inconsistent state and " + + "should be restarted. To minimize cluster down time, enabling " + + "resource manager high avaliability is recommended."; + private final String resource; - public ResourceNotFoundException(String message) { - super(message); + public ResourceNotFoundException(String resource) { + super(String.format(MESSAGE, resource)); + this.resource = resource; } - public ResourceNotFoundException(Throwable cause) { - super(cause); + public ResourceNotFoundException(String resource, Throwable cause) { + super(String.format(MESSAGE, resource), cause); + this.resource = resource; } - public ResourceNotFoundException(String message, Throwable cause) { - super(message, cause); + /** + * Return the resource that was not found. + * @return the resource that was not found + */ + public String getResource() { + return resource; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 561deb35b19..e56bceecc20 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -31,8 +31,8 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto; -import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.UnitsConversionUtil; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import java.util.Arrays; import java.util.Map; @@ -174,8 +174,7 @@ public void setResourceInformation(String resource, } @Override - public void setResourceValue(String resource, Long value) - throws ResourceNotFoundException { + public void setResourceValue(String resource, Long value) { maybeInitBuilder(); if (resource == null) { throw new IllegalArgumentException("resource type object cannot be null"); @@ -190,15 +189,13 @@ public void setResourceValue(String resource, Long value) } @Override - public ResourceInformation getResourceInformation(String resource) - throws ResourceNotFoundException { + public ResourceInformation getResourceInformation(String resource) { initResources(); return super.getResourceInformation(resource); } @Override - public long getResourceValue(String resource) - throws ResourceNotFoundException { + public long getResourceValue(String resource) { initResources(); return super.getResourceValue(resource); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index ea5c8a8c3f6..74bdec43b7e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -76,22 +76,18 @@ private int compare(Resource lhs, Resource rhs) { int ret = 0; for (String rName : resourceNames) { - try { - ResourceInformation lhsResourceInformation = - lhs.getResourceInformation(rName); - ResourceInformation rhsResourceInformation = - rhs.getResourceInformation(rName); - int diff = lhsResourceInformation.compareTo(rhsResourceInformation); - if (diff >= 1) { - lhsGreater = true; - } else if (diff <= -1) { - rhsGreater = true; - } - } catch (ResourceNotFoundException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + rName, ye); + ResourceInformation lhsResourceInformation = + lhs.getResourceInformation(rName); + ResourceInformation rhsResourceInformation = + rhs.getResourceInformation(rName); + int diff = lhsResourceInformation.compareTo(rhsResourceInformation); + if (diff >= 1) { + lhsGreater = true; + } else if (diff <= -1) { + rhsGreater = true; } } + if (lhsGreater && rhsGreater) { ret = 0; } else if (lhsGreater) { @@ -148,25 +144,21 @@ protected float getResourceAsValue(Resource clusterResource, float min = Float.MAX_VALUE; float max = 0.0f; for (String rName : resourceNames) { - try { - ResourceInformation clusterResourceResourceInformation = - clusterResource.getResourceInformation(rName); - ResourceInformation resourceInformation = - resource.getResourceInformation(rName); - long resourceValue = UnitsConversionUtil - .convert(resourceInformation.getUnits(), - clusterResourceResourceInformation.getUnits(), - resourceInformation.getValue()); - float tmp = - (float) resourceValue / (float) clusterResourceResourceInformation - .getValue(); - min = min < tmp ? min : tmp; - max = max > tmp ? max : tmp; - } catch (ResourceNotFoundException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); - } + ResourceInformation clusterResourceResourceInformation = + clusterResource.getResourceInformation(rName); + ResourceInformation resourceInformation = + resource.getResourceInformation(rName); + long resourceValue = UnitsConversionUtil + .convert(resourceInformation.getUnits(), + clusterResourceResourceInformation.getUnits(), + resourceInformation.getValue()); + float tmp = + (float) resourceValue / (float) clusterResourceResourceInformation + .getValue(); + min = min < tmp ? min : tmp; + max = max > tmp ? max : tmp; } + return (dominant) ? max : min; } @@ -174,24 +166,19 @@ protected float getResourceAsValue(Resource clusterResource, public long computeAvailableContainers(Resource available, Resource required) { long min = Long.MAX_VALUE; for (String resource : resourceNames) { - try { - ResourceInformation availableResource = - available.getResourceInformation(resource); - ResourceInformation requiredResource = - required.getResourceInformation(resource); - long requiredResourceValue = UnitsConversionUtil - .convert(requiredResource.getUnits(), availableResource.getUnits(), - requiredResource.getValue()); - if (requiredResourceValue != 0) { - long tmp = availableResource.getValue() / requiredResourceValue; - min = min < tmp ? min : tmp; - } - } catch (ResourceNotFoundException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); + ResourceInformation availableResource = + available.getResourceInformation(resource); + ResourceInformation requiredResource = + required.getResourceInformation(resource); + long requiredResourceValue = UnitsConversionUtil + .convert(requiredResource.getUnits(), availableResource.getUnits(), + requiredResource.getValue()); + if (requiredResourceValue != 0) { + long tmp = availableResource.getValue() / requiredResourceValue; + min = min < tmp ? min : tmp; } - } + return min > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) min; } @@ -210,6 +197,7 @@ public boolean isInvalidDivisor(Resource r) { return true; } } + return false; } @@ -217,23 +205,19 @@ public boolean isInvalidDivisor(Resource r) { public float ratio(Resource a, Resource b) { float ratio = 0.0f; for (String resource : resourceNames) { - try { - ResourceInformation aResourceInformation = - a.getResourceInformation(resource); - ResourceInformation bResourceInformation = - b.getResourceInformation(resource); - long bResourceValue = UnitsConversionUtil - .convert(bResourceInformation.getUnits(), - aResourceInformation.getUnits(), - bResourceInformation.getValue()); - float tmp = - (float) aResourceInformation.getValue() / (float) bResourceValue; - ratio = ratio > tmp ? ratio : tmp; - } catch (ResourceNotFoundException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); - } + ResourceInformation aResourceInformation = + a.getResourceInformation(resource); + ResourceInformation bResourceInformation = + b.getResourceInformation(resource); + long bResourceValue = UnitsConversionUtil + .convert(bResourceInformation.getUnits(), + aResourceInformation.getUnits(), + bResourceInformation.getValue()); + float tmp = + (float) aResourceInformation.getValue() / (float) bResourceValue; + ratio = ratio > tmp ? ratio : tmp; } + return ratio; } @@ -245,16 +229,12 @@ public Resource divideAndCeil(Resource numerator, int denominator) { public Resource divideAndCeil(Resource numerator, long denominator) { Resource ret = Resource.newInstance(numerator); for (String resource : resourceNames) { - try { - ResourceInformation resourceInformation = - ret.getResourceInformation(resource); - resourceInformation.setValue( - divideAndCeil(resourceInformation.getValue(), denominator)); - } catch (ResourceNotFoundException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); - } + ResourceInformation resourceInformation = + ret.getResourceInformation(resource); + resourceInformation.setValue( + divideAndCeil(resourceInformation.getValue(), denominator)); } + return ret; } @@ -271,41 +251,37 @@ public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { - try { - ResourceInformation rResourceInformation = - r.getResourceInformation(resource); - ResourceInformation minimumResourceInformation = - minimumResource.getResourceInformation(resource); - ResourceInformation maximumResourceInformation = - maximumResource.getResourceInformation(resource); - ResourceInformation stepFactorResourceInformation = - stepFactor.getResourceInformation(resource); - ResourceInformation tmp = ret.getResourceInformation(resource); - - long rValue = rResourceInformation.getValue(); - long minimumValue = UnitsConversionUtil - .convert(minimumResourceInformation.getUnits(), - rResourceInformation.getUnits(), - minimumResourceInformation.getValue()); - long maximumValue = UnitsConversionUtil - .convert(maximumResourceInformation.getUnits(), - rResourceInformation.getUnits(), - maximumResourceInformation.getValue()); - long stepFactorValue = UnitsConversionUtil - .convert(stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); - long value = Math.max(rValue, minimumValue); - if (stepFactorValue != 0) { - value = roundUp(value, stepFactorValue); - } - tmp.setValue(Math.min(value, maximumValue)); - ret.setResourceInformation(resource, tmp); - } catch (ResourceNotFoundException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); + ResourceInformation rResourceInformation = + r.getResourceInformation(resource); + ResourceInformation minimumResourceInformation = + minimumResource.getResourceInformation(resource); + ResourceInformation maximumResourceInformation = + maximumResource.getResourceInformation(resource); + ResourceInformation stepFactorResourceInformation = + stepFactor.getResourceInformation(resource); + ResourceInformation tmp = ret.getResourceInformation(resource); + + long rValue = rResourceInformation.getValue(); + long minimumValue = UnitsConversionUtil + .convert(minimumResourceInformation.getUnits(), + rResourceInformation.getUnits(), + minimumResourceInformation.getValue()); + long maximumValue = UnitsConversionUtil + .convert(maximumResourceInformation.getUnits(), + rResourceInformation.getUnits(), + maximumResourceInformation.getValue()); + long stepFactorValue = UnitsConversionUtil + .convert(stepFactorResourceInformation.getUnits(), + rResourceInformation.getUnits(), + stepFactorResourceInformation.getValue()); + long value = Math.max(rValue, minimumValue); + if (stepFactorValue != 0) { + value = roundUp(value, stepFactorValue); } + tmp.setValue(Math.min(value, maximumValue)); + ret.setResourceInformation(resource, tmp); } + return ret; } @@ -322,30 +298,26 @@ public Resource roundDown(Resource r, Resource stepFactor) { private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) { Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { - try { - ResourceInformation rResourceInformation = - r.getResourceInformation(resource); - ResourceInformation stepFactorResourceInformation = - stepFactor.getResourceInformation(resource); - - long rValue = rResourceInformation.getValue(); - long stepFactorValue = UnitsConversionUtil - .convert(stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); - long value = rValue; - if (stepFactorValue != 0) { - value = roundUp ? roundUp(rValue, stepFactorValue) : - roundDown(rValue, stepFactorValue); - } - ResourceInformation - .copy(rResourceInformation, ret.getResourceInformation(resource)); - ret.getResourceInformation(resource).setValue(value); - } catch (ResourceNotFoundException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); + ResourceInformation rResourceInformation = + r.getResourceInformation(resource); + ResourceInformation stepFactorResourceInformation = + stepFactor.getResourceInformation(resource); + + long rValue = rResourceInformation.getValue(); + long stepFactorValue = UnitsConversionUtil + .convert(stepFactorResourceInformation.getUnits(), + rResourceInformation.getUnits(), + stepFactorResourceInformation.getValue()); + long value = rValue; + if (stepFactorValue != 0) { + value = roundUp ? roundUp(rValue, stepFactorValue) : + roundDown(rValue, stepFactorValue); } + ResourceInformation + .copy(rResourceInformation, ret.getResourceInformation(resource)); + ret.getResourceInformation(resource).setValue(value); } + return ret; } @@ -365,56 +337,49 @@ private Resource multiplyAndNormalize(Resource r, double by, Resource stepFactor, boolean roundUp) { Resource ret = Resource.newInstance(r); for (String resource : resourceNames) { - try { - ResourceInformation rResourceInformation = r - .getResourceInformation(resource); - ResourceInformation stepFactorResourceInformation = stepFactor - .getResourceInformation(resource); - ResourceInformation tmp = ret.getResourceInformation(resource); - - long rValue = rResourceInformation.getValue(); - long stepFactorValue = UnitsConversionUtil.convert( - stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); - long value; - if (stepFactorValue != 0) { - value = roundUp - ? roundUp((long) Math.ceil(rValue * by), stepFactorValue) - : roundDown((long) (rValue * by), stepFactorValue); - } else { - value = roundUp - ? (long) Math.ceil(rValue * by) - : (long) (rValue * by); - } - tmp.setValue(value); - } catch (ResourceNotFoundException ye) { - throw new IllegalArgumentException( - "Error getting resource information for " + resource, ye); + ResourceInformation rResourceInformation = r + .getResourceInformation(resource); + ResourceInformation stepFactorResourceInformation = stepFactor + .getResourceInformation(resource); + ResourceInformation tmp = ret.getResourceInformation(resource); + + long rValue = rResourceInformation.getValue(); + long stepFactorValue = UnitsConversionUtil.convert( + stepFactorResourceInformation.getUnits(), + rResourceInformation.getUnits(), + stepFactorResourceInformation.getValue()); + long value; + if (stepFactorValue != 0) { + value = roundUp + ? roundUp((long) Math.ceil(rValue * by), stepFactorValue) + : roundDown((long) (rValue * by), stepFactorValue); + } else { + value = roundUp + ? (long) Math.ceil(rValue * by) + : (long) (rValue * by); } + tmp.setValue(value); } + return ret; } @Override public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) { for (String resource : resourceNames) { - try { - ResourceInformation sResourceInformation = - smaller.getResourceInformation(resource); - ResourceInformation bResourceInformation = - bigger.getResourceInformation(resource); - long sResourceValue = UnitsConversionUtil - .convert(sResourceInformation.getUnits(), - bResourceInformation.getUnits(), - sResourceInformation.getValue()); - if(sResourceValue > bResourceInformation.getValue()) { - return false; - } - } catch (ResourceNotFoundException ye) { + ResourceInformation sResourceInformation = + smaller.getResourceInformation(resource); + ResourceInformation bResourceInformation = + bigger.getResourceInformation(resource); + long sResourceValue = UnitsConversionUtil + .convert(sResourceInformation.getUnits(), + bResourceInformation.getUnits(), + sResourceInformation.getValue()); + if(sResourceValue > bResourceInformation.getValue()) { return false; } } + return true; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index f62114dfc9b..0d16302805f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -98,14 +98,12 @@ public void setVirtualCores(int virtualCores) { @Override public void setResourceInformation(String resource, - ResourceInformation resourceInformation) - throws ResourceNotFoundException { + ResourceInformation resourceInformation) { throw new RuntimeException(name + " cannot be modified!"); } @Override - public void setResourceValue(String resource, Long value) - throws ResourceNotFoundException { + public void setResourceValue(String resource, Long value) { throw new RuntimeException(name + " cannot be modified!"); }