diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index a5b40210ff4..2aa9a5c0bb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -617,7 +617,7 @@ - + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 04579c53e93..7bda8f59818 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.impl.BaseResource; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.util.Records; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.ResourceUtils; /** @@ -59,8 +60,15 @@ @Stable public abstract class Resource implements Comparable { - protected static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); - protected static final String VCORES = ResourceInformation.VCORES.getName(); + protected ResourceInformation[] resources = null; + + // Number of mandatory resources, this is added to avoid invoke + // MandatoryResources.values().length, since values() internally will + // copy array, etc. + protected static final int NUM_MANDATORY_RESOURCES = 2; + + protected static final int MEMORY_INDEX = 0; + protected static final int VCORES_INDEX = 1; @Public @Stable @@ -201,7 +209,9 @@ public void setMemorySize(long memory) { */ @Public @Evolving - public abstract ResourceInformation[] getResources(); + public ResourceInformation[] getResources() { + return resources; + } /** * Get ResourceInformation for a specified resource. @@ -236,12 +246,14 @@ public ResourceInformation getResourceInformation(String resource) @Evolving public ResourceInformation getResourceInformation(int index) throws ResourceNotFoundException { - ResourceInformation[] resources = getResources(); - if (index < 0 || index >= resources.length) { - throw new ResourceNotFoundException("Unknown resource at index '" + index - + "'. Vaid resources are: " + Arrays.toString(resources)); + ResourceInformation ri = null; + try { + ResourceInformation[] resources = getResources(); + ri = resources[index]; + } catch (ArrayIndexOutOfBoundsException e) { + checkIndexAndThrowExceptionWhenArrayOutOfBound(index); } - return resources[index]; + return ri; } /** @@ -271,11 +283,11 @@ public long getResourceValue(String resource) public void setResourceInformation(String resource, ResourceInformation resourceInformation) throws ResourceNotFoundException { - if (resource.equals(MEMORY)) { + if (resource.equals(ResourceInformation.MEMORY_URI)) { this.setMemorySize(resourceInformation.getValue()); return; } - if (resource.equals(VCORES)) { + if (resource.equals(ResourceInformation.VCORES_URI)) { this.setVirtualCores((int) resourceInformation.getValue()); return; } @@ -318,11 +330,11 @@ public void setResourceInformation(int index, @Evolving public void setResourceValue(String resource, long value) throws ResourceNotFoundException { - if (resource.equals(MEMORY)) { + if (resource.equals(ResourceInformation.MEMORY_URI)) { this.setMemorySize(value); return; } - if (resource.equals(VCORES)) { + if (resource.equals(ResourceInformation.VCORES_URI)) { this.setVirtualCores((int)value); return; } @@ -346,27 +358,21 @@ public void setResourceValue(String resource, long value) @Evolving public void setResourceValue(int index, long value) throws ResourceNotFoundException { - ResourceInformation[] resources = getResources(); - if (index < 0 || index >= resources.length) { - throw new ResourceNotFoundException("Unknown resource at index '" + index - + "'. Valid resources are " + Arrays.toString(resources)); + try { + ResourceInformation[] resources = getResources(); + resources[index].setValue(value); + } catch (ArrayIndexOutOfBoundsException e) { + checkIndexAndThrowExceptionWhenArrayOutOfBound(index); } - resources[index].setValue(value); } - @Override - public int hashCode() { - final int prime = 263167; - - int result = (int) (939769357 - + getMemorySize()); // prime * result = 939769357 initially - result = prime * result + getVirtualCores(); - for (ResourceInformation entry : getResources()) { - if (!entry.getName().equals(MEMORY) && !entry.getName().equals(VCORES)) { - result = prime * result + entry.hashCode(); - } + private void checkIndexAndThrowExceptionWhenArrayOutOfBound(int index) { + if (index < 0) { + throw new ResourceNotFoundException("Specified index must >= 0"); + } else { + throw new ResourceNotFoundException( + "Specified index beyond known #resource-types"); } - return result; } @Override @@ -381,10 +387,6 @@ public boolean equals(Object obj) { return false; } Resource other = (Resource) obj; - if (getMemorySize() != other.getMemorySize() - || getVirtualCores() != other.getVirtualCores()) { - return false; - } ResourceInformation[] myVectors = getResources(); ResourceInformation[] otherVectors = other.getResources(); @@ -404,62 +406,70 @@ public boolean equals(Object obj) { } @Override + public int compareTo(Resource other) { + ResourceInformation[] thisResources = this.getResources(); + ResourceInformation[] otherResources = other.getResources(); + + int arrLenThis = this.resources.length; + int arrLenOther = otherResources.length; + + // compare memory and vcores first(in that order) to preserve + // existing behaviour + for (int i = 0; i < arrLenThis; i++) { + if (i >= arrLenOther) { + // For two vectors with different size and same prefix. Shorter vector + // goes first. + return 1; + } + + ResourceInformation entry = thisResources[i]; + ResourceInformation otherEntry = otherResources[i]; + long diff = entry.compareTo(otherEntry); + if (diff != 0) { + return (int) Math.signum(diff); + } + } + + if (arrLenThis < arrLenOther) { + return -1; + } + + return 0; + } + + @Override public String toString() { StringBuilder sb = new StringBuilder(); - sb.append(""); return sb.toString(); } @Override - public int compareTo(Resource other) { - ResourceInformation[] thisResources = this.getResources(); - ResourceInformation[] otherResources = other.getResources(); - - // compare memory and vcores first(in that order) to preserve - // existing behaviour - long diff = this.getMemorySize() - other.getMemorySize(); - if (diff == 0) { - diff = this.getVirtualCores() - other.getVirtualCores(); - } - if (diff == 0) { - diff = thisResources.length - otherResources.length; - if (diff == 0) { - int maxLength = ResourceUtils.getResourceTypesArray().length; - for (int i = 0; i < maxLength; i++) { - // For memory and vcores, we can skip the loop as it's already - // compared. - if (i < 2) { - continue; - } - - ResourceInformation entry = thisResources[i]; - ResourceInformation otherEntry = otherResources[i]; - if (entry.getName().equals(otherEntry.getName())) { - diff = entry.compareTo(otherEntry); - if (diff != 0) { - break; - } - } - } - } + public int hashCode() { + final int prime = 263167; + int result = 939769357; + for (ResourceInformation entry : getResources()) { + result = prime * result + entry.hashCode(); } - return Long.compare(diff, 0); + return result; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 3ab7ccd04d3..2a040948d58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.api.records; +import org.apache.curator.shaded.com.google.common.reflect.ClassPath; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.util.UnitsConversionUtil; @@ -34,8 +36,8 @@ private long minimumAllocation; private long maximumAllocation; - private static final String MEMORY_URI = "memory-mb"; - private static final String VCORES_URI = "vcores"; + public static final String MEMORY_URI = "memory-mb"; + public static final String VCORES_URI = "vcores"; public static final ResourceInformation MEMORY_MB = ResourceInformation.newInstance(MEMORY_URI, "Mi"); @@ -84,6 +86,16 @@ public void setUnits(String rUnits) { } /** + * Checking if a unit included by KNOWN_UNITS is an expensive operation. This + * can be avoided in critical path in RM. + * @param rUnits units for the resource + */ + @InterfaceAudience.Private + public void setUnitsWithoutValidation(String rUnits) { + this.units = rUnits; + } + + /** * Get the resource type. * * @return the resource type diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java index b5cc4d63e4d..2cbbb4447c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java @@ -18,13 +18,18 @@ package org.apache.hadoop.yarn.api.records.impl; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import java.util.Arrays; +import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_MB; + /** *

* BaseResource extends Resource to handle base resources such @@ -54,48 +59,46 @@ * * @see Resource */ -@Public +@InterfaceAudience.Private @Unstable public class BaseResource extends Resource { private ResourceInformation memoryResInfo; private ResourceInformation vcoresResInfo; - protected ResourceInformation[] resources = null; - protected ResourceInformation[] readOnlyResources = null; - - // Number of mandatory resources, this is added to avoid invoke - // MandatoryResources.values().length, since values() internally will - // copy array, etc. - private static final int NUM_MANDATORY_RESOURCES = 2; - - protected enum MandatoryResources { - MEMORY(0), VCORES(1); - - private final int id; - - MandatoryResources(int id) { - this.id = id; - } - - public int getId() { - return this.id; - } - } public BaseResource() { // Base constructor. } public BaseResource(long memory, long vcores) { - this.memoryResInfo = ResourceInformation.newInstance(MEMORY, - ResourceInformation.MEMORY_MB.getUnits(), memory); - this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores); + this.memoryResInfo = BaseResource.newDefaultMemoryInformation(memory); + this.vcoresResInfo = BaseResource.newDefaultVCoresInformation(vcores); resources = new ResourceInformation[NUM_MANDATORY_RESOURCES]; - readOnlyResources = new ResourceInformation[NUM_MANDATORY_RESOURCES]; - resources[MandatoryResources.MEMORY.id] = memoryResInfo; - resources[MandatoryResources.VCORES.id] = vcoresResInfo; - readOnlyResources = Arrays.copyOf(resources, resources.length); + resources[MEMORY_INDEX] = memoryResInfo; + resources[VCORES_INDEX] = vcoresResInfo; + } + + public static ResourceInformation newDefaultMemoryInformation(long value) { + ResourceInformation ri = new ResourceInformation(); + ri.setName(ResourceInformation.MEMORY_URI); + ri.setValue(value); + ri.setResourceType(ResourceTypes.COUNTABLE); + ri.setUnitsWithoutValidation(MEMORY_MB.getUnits()); + ri.setMinimumAllocation(0); + ri.setMaximumAllocation(Long.MAX_VALUE); + return ri; + } + + public static ResourceInformation newDefaultVCoresInformation(long value) { + ResourceInformation ri = new ResourceInformation(); + ri.setName(ResourceInformation.VCORES_URI); + ri.setValue(value); + ri.setResourceType(ResourceTypes.COUNTABLE); + ri.setUnitsWithoutValidation(""); + ri.setMinimumAllocation(0); + ri.setMaximumAllocation(Long.MAX_VALUE); + return ri; } @Override @@ -131,7 +134,53 @@ public void setVirtualCores(int vcores) { } @Override - public ResourceInformation[] getResources() { - return readOnlyResources; + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof Resource)) { + return false; + } + Resource other = (Resource) obj; + if (getMemorySize() != other.getMemorySize() + || getVirtualCores() != other.getVirtualCores()) { + return false; + } + + return true; + } + + @Override + public int compareTo(Resource other) { + // compare memory and vcores first(in that order) to preserve + // existing behaviour + long diff = this.getMemorySize() - other.getMemorySize(); + if (diff == 0) { + return this.getVirtualCores() - other.getVirtualCores(); + } + return (int) Math.signum(diff); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(""); + return sb.toString(); + } + + @Override + public int hashCode() { + final int prime = 263167; + + // prime * result = 939769357 initially + int result = (int) (939769357 + getMemorySize()); + result = prime * result + getVirtualCores(); + + return result; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index ff22dd019c3..7ff71e639fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -40,7 +40,7 @@ @Private @Unstable -public class ResourcePBImpl extends BaseResource { +public class ResourcePBImpl extends Resource { private static final Log LOG = LogFactory.getLog(ResourcePBImpl.class); @@ -95,7 +95,7 @@ public int getMemory() { @Override public long getMemorySize() { // memory should always be present - ResourceInformation ri = resources[MandatoryResources.MEMORY.getId()]; + ResourceInformation ri = resources[MEMORY_INDEX]; if (ri.getUnits().isEmpty()) { return ri.getValue(); @@ -119,7 +119,7 @@ public void setMemorySize(long memory) { @Override public int getVirtualCores() { // vcores should always be present - return (int) resources[MandatoryResources.VCORES.getId()].getValue(); + return (int) resources[VCORES_INDEX].getValue(); } @Override @@ -156,7 +156,6 @@ private void initResources() { resources[index].setValue(value); } } - readOnlyResources = Arrays.copyOf(resources, resources.length); this.setMemorySize(p.getMemory()); this.setVirtualCores(p.getVirtualCores()); } @@ -187,11 +186,6 @@ public void setResourceValue(String resource, long value) } @Override - public ResourceInformation[] getResources() { - return super.getResources(); - } - - @Override public ResourceInformation getResourceInformation(String resource) throws ResourceNotFoundException { return super.getResourceInformation(resource); @@ -212,7 +206,6 @@ private void initResourcesMap() { } resources = new ResourceInformation[types.length]; - readOnlyResources = new ResourceInformation[types.length]; for (ResourceInformation entry : types) { int index = ResourceUtils.getResourceTypeIndex().get(entry.getName()); resources[index] = ResourceInformation.newInstance(entry); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index ffd4fec0ee0..2bb378cede0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -73,7 +73,7 @@ private int compare(Resource lhs, Resource rhs) { boolean rhsGreater = false; int ret = 0; - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation lhsResourceInformation = lhs .getResourceInformation(i); @@ -111,10 +111,12 @@ public int compare(Resource clusterResource, Resource lhs, Resource rhs, // resources and then look for which resource has the biggest // share overall. ResourceInformation[] clusterRes = clusterResource.getResources(); + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + // If array creation shows up as a time sink, these arrays could be cached // because they're always the same length. - double[] lhsShares = new double[clusterRes.length]; - double[] rhsShares = new double[clusterRes.length]; + double[] lhsShares = new double[maxLength]; + double[] rhsShares = new double[maxLength]; double diff; try { @@ -124,11 +126,11 @@ public int compare(Resource clusterResource, Resource lhs, Resource rhs, calculateShares(clusterRes, lhs, rhs, lhsShares, rhsShares, max); diff = max[0] - max[1]; - } else if (clusterRes.length == 2) { + } else if (maxLength == 2) { // Special case to handle the common scenario of only CPU and memory // so that we can optimize for performance - diff = calculateSharesForMandatoryResources(clusterRes, lhs, rhs, - lhsShares, rhsShares); + diff = calculateSharesForTwoMandatoryResources(clusterRes, lhs, rhs, + lhsShares, rhsShares, maxLength); } else { calculateShares(clusterRes, lhs, rhs, lhsShares, rhsShares); @@ -182,7 +184,8 @@ private void calculateShares(ResourceInformation[] clusterRes, Resource first, ResourceInformation[] firstRes = first.getResources(); ResourceInformation[] secondRes = second.getResources(); - for (int i = 0; i < clusterRes.length; i++) { + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { firstShares[i] = calculateShare(clusterRes[i], firstRes[i]); secondShares[i] = calculateShare(clusterRes[i], secondRes[i]); } @@ -200,14 +203,15 @@ private void calculateShares(ResourceInformation[] clusterRes, Resource first, * @param second the second resource to compare * @param firstShares an array to store the shares for the first resource * @param secondShares an array to store the shares for the second resource + * @param nKnownResourceTypes number of known resource types * @return -1.0, 0.0, or 1.0, depending on whether the max share of the first * resource is less than, equal to, or greater than the max share of the * second resource, respectively * @throws NullPointerException if any parameter is null */ - private int calculateSharesForMandatoryResources( + private int calculateSharesForTwoMandatoryResources( ResourceInformation[] clusterRes, Resource first, Resource second, - double[] firstShares, double[] secondShares) { + double[] firstShares, double[] secondShares, int nKnownResourceTypes) { ResourceInformation[] firstRes = first.getResources(); ResourceInformation[] secondRes = second.getResources(); int firstDom = 0; @@ -215,7 +219,7 @@ private int calculateSharesForMandatoryResources( int firstSub = 0; int secondSub = 0; - for (int i = 0; i < clusterRes.length; i++) { + for (int i = 0; i < nKnownResourceTypes; i++) { firstShares[i] = calculateShare(clusterRes[i], firstRes[i]); secondShares[i] = calculateShare(clusterRes[i], secondRes[i]); @@ -280,7 +284,8 @@ private void calculateShares(ResourceInformation[] clusterRes, Resource first, max[0] = 0.0; max[1] = 0.0; - for (int i = 0; i < clusterRes.length; i++) { + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { firstShares[i] = calculateShare(clusterRes[i], firstRes[i]); secondShares[i] = calculateShare(clusterRes[i], secondRes[i]); @@ -339,7 +344,7 @@ private double compareShares(double[] lhsShares, double[] rhsShares) { public long computeAvailableContainers(Resource available, Resource required) { long min = Long.MAX_VALUE; - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation availableResource = available .getResourceInformation(i); @@ -358,11 +363,12 @@ public long computeAvailableContainers(Resource available, @Override public float divide(Resource clusterResource, Resource numerator, Resource denominator) { + int nKnownResourceTypes = ResourceUtils.getNumberOfKnownResourceTypes(); ResourceInformation[] clusterRes = clusterResource.getResources(); // We have to provide the calculateShares() method with somewhere to store // the shares. We don't actually need these shares afterwards. - double[] numeratorShares = new double[clusterRes.length]; - double[] denominatorShares = new double[clusterRes.length]; + double[] numeratorShares = new double[nKnownResourceTypes]; + double[] denominatorShares = new double[nKnownResourceTypes]; // We also have to provide a place for calculateShares() to store the max // shares so that we can use them. double[] max = new double[2]; @@ -386,7 +392,7 @@ public boolean isInvalidDivisor(Resource r) { @Override public float ratio(Resource a, Resource b) { float ratio = 0.0f; - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation aResourceInformation = a.getResourceInformation(i); ResourceInformation bResourceInformation = b.getResourceInformation(i); @@ -407,7 +413,7 @@ public Resource divideAndCeil(Resource numerator, int denominator) { public Resource divideAndCeil(Resource numerator, long denominator) { Resource ret = Resource.newInstance(numerator); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation resourceInformation = ret.getResourceInformation(i); resourceInformation @@ -428,7 +434,7 @@ public Resource divideAndCeil(Resource numerator, float denominator) { public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { Resource ret = Resource.newInstance(r); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation minimumResourceInformation = minimumResource @@ -474,7 +480,7 @@ public Resource roundDown(Resource r, Resource stepFactor) { private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) { Resource ret = Resource.newInstance(r); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation stepFactorResourceInformation = stepFactor @@ -513,7 +519,7 @@ public Resource multiplyAndNormalizeDown(Resource r, double by, private Resource multiplyAndNormalize(Resource r, double by, Resource stepFactor, boolean roundUp) { Resource ret = Resource.newInstance(r); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation stepFactorResourceInformation = stepFactor @@ -542,7 +548,7 @@ private Resource multiplyAndNormalize(Resource r, double by, @Override public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation sResourceInformation = smaller .getResourceInformation(i); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 1e2ce155505..325bce465b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -24,12 +24,9 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; -import org.apache.hadoop.yarn.api.records.impl.BaseResource; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.util.UnitsConversionUtil; -import java.util.Arrays; - /** * Resources is a computation class which provides a set of apis to do * mathematical operations on Resource object. @@ -45,9 +42,11 @@ * Helper class to create a resource with a fixed value for all resource * types. For example, a NONE resource which returns 0 for any resource type. */ - static class FixedValueResource extends BaseResource { + @InterfaceAudience.Private + @Unstable + static class FixedValueResource extends Resource { - private long resourceValue; + private final long resourceValue; private String name; /** @@ -101,6 +100,19 @@ public void setVirtualCores(int virtualCores) { } @Override + public void setResourceInformation(int index, + ResourceInformation resourceInformation) + throws ResourceNotFoundException { + throw new RuntimeException(name + " cannot be modified!"); + } + + @Override + public void setResourceValue(int index, long value) + throws ResourceNotFoundException { + throw new RuntimeException(name + " cannot be modified!"); + } + + @Override public void setResourceInformation(String resource, ResourceInformation resourceInformation) throws ResourceNotFoundException { @@ -117,19 +129,11 @@ private void initResourceMap() { ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); if (types != null) { resources = new ResourceInformation[types.length]; - readOnlyResources = new ResourceInformation[types.length]; for (int index = 0; index < types.length; index++) { resources[index] = ResourceInformation.newInstance(types[index]); resources[index].setValue(resourceValue); - - // this is a fix for getVirtualCores returning an int - if (resourceValue > Integer.MAX_VALUE && ResourceInformation.VCORES - .getName().equals(resources[index].getName())) { - resources[index].setValue((long) Integer.MAX_VALUE); - } } } - readOnlyResources = Arrays.copyOf(resources, resources.length); } }