From f15acb93f8373ded34e7ceb04826d7168a3396cc Mon Sep 17 00:00:00 2001 From: Sunil G Date: Fri, 1 Sep 2017 19:51:16 +0530 Subject: [PATCH] YARN-7136.YARN-3926 --- .../hadoop-yarn/dev-support/findbugs-exclude.xml | 6 + .../apache/hadoop/yarn/api/records/Resource.java | 151 +++++++++++---------- .../yarn/api/records/ResourceInformation.java | 16 ++- .../hadoop/yarn/api/records/impl/BaseResource.java | 111 ++++++++++----- .../yarn/api/records/impl/pb/ResourcePBImpl.java | 8 +- .../util/resource/DominantResourceCalculator.java | 38 +++--- .../hadoop/yarn/util/resource/Resources.java | 35 +++-- 7 files changed, 224 insertions(+), 141 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index a5b4021..5a49e79 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -621,4 +621,10 @@ + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 04579c5..eb1a345 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -59,9 +59,18 @@ @Stable public abstract class Resource implements Comparable { + protected ResourceInformation[] resources = null; protected static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); protected static final String VCORES = ResourceInformation.VCORES.getName(); + // Number of mandatory resources, this is added to avoid invoke + // MandatoryResources.values().length, since values() internally will + // copy array, etc. + protected static final int NUM_MANDATORY_RESOURCES = 2; + + protected static final int MEMORY_INDEX = 0; + protected static final int VCORES_INDEX = 1; + @Public @Stable public static Resource newInstance(int memory, int vCores) { @@ -201,7 +210,9 @@ public void setMemorySize(long memory) { */ @Public @Evolving - public abstract ResourceInformation[] getResources(); + public ResourceInformation[] getResources() { + return resources; + } /** * Get ResourceInformation for a specified resource. @@ -236,12 +247,14 @@ public ResourceInformation getResourceInformation(String resource) @Evolving public ResourceInformation getResourceInformation(int index) throws ResourceNotFoundException { - ResourceInformation[] resources = getResources(); - if (index < 0 || index >= resources.length) { - throw new ResourceNotFoundException("Unknown resource at index '" + index - + "'. Vaid resources are: " + Arrays.toString(resources)); + ResourceInformation ri = null; + try { + ResourceInformation[] resources = getResources(); + ri = resources[index]; + } catch (ArrayIndexOutOfBoundsException e) { + checkIndexAndThrowExceptionWhenArrayOutOfBound(index); } - return resources[index]; + return ri; } /** @@ -346,27 +359,21 @@ public void setResourceValue(String resource, long value) @Evolving public void setResourceValue(int index, long value) throws ResourceNotFoundException { - ResourceInformation[] resources = getResources(); - if (index < 0 || index >= resources.length) { - throw new ResourceNotFoundException("Unknown resource at index '" + index - + "'. Valid resources are " + Arrays.toString(resources)); + try { + ResourceInformation[] resources = getResources(); + resources[index].setValue(value); + } catch (ArrayIndexOutOfBoundsException e) { + checkIndexAndThrowExceptionWhenArrayOutOfBound(index); } - resources[index].setValue(value); } - @Override - public int hashCode() { - final int prime = 263167; - - int result = (int) (939769357 - + getMemorySize()); // prime * result = 939769357 initially - result = prime * result + getVirtualCores(); - for (ResourceInformation entry : getResources()) { - if (!entry.getName().equals(MEMORY) && !entry.getName().equals(VCORES)) { - result = prime * result + entry.hashCode(); - } + private void checkIndexAndThrowExceptionWhenArrayOutOfBound(int index) { + if (index < 0) { + throw new ResourceNotFoundException("Specified index must >= 0"); + } else { + throw new ResourceNotFoundException( + "Specified index beyond known #resource-types"); } - return result; } @Override @@ -381,10 +388,6 @@ public boolean equals(Object obj) { return false; } Resource other = (Resource) obj; - if (getMemorySize() != other.getMemorySize() - || getVirtualCores() != other.getVirtualCores()) { - return false; - } ResourceInformation[] myVectors = getResources(); ResourceInformation[] otherVectors = other.getResources(); @@ -404,62 +407,62 @@ public boolean equals(Object obj) { } @Override + public int compareTo(Resource other) { + ResourceInformation[] thisResources = this.getResources(); + ResourceInformation[] otherResources = other.getResources(); + + int arrLenThis = this.resources.length; + int arrLenOther = otherResources.length; + + // compare memory and vcores first(in that order) to preserve + // existing behaviour + for (int i = 0; i < arrLenThis; i++) { + if (i >= arrLenOther) { + // For two vectors with different size and same prefix. Shorter vector + // goes first. + return 1; + } + + ResourceInformation entry = thisResources[i]; + ResourceInformation otherEntry = otherResources[i]; + long diff = entry.compareTo(otherEntry); + if (diff != 0) { + return (int) Math.signum(diff); + } + } + + if (arrLenThis < arrLenOther) { + return -1; + } + + return 0; + } + + @Override public String toString() { StringBuilder sb = new StringBuilder(); - sb.append(""); return sb.toString(); } @Override - public int compareTo(Resource other) { - ResourceInformation[] thisResources = this.getResources(); - ResourceInformation[] otherResources = other.getResources(); - - // compare memory and vcores first(in that order) to preserve - // existing behaviour - long diff = this.getMemorySize() - other.getMemorySize(); - if (diff == 0) { - diff = this.getVirtualCores() - other.getVirtualCores(); - } - if (diff == 0) { - diff = thisResources.length - otherResources.length; - if (diff == 0) { - int maxLength = ResourceUtils.getResourceTypesArray().length; - for (int i = 0; i < maxLength; i++) { - // For memory and vcores, we can skip the loop as it's already - // compared. - if (i < 2) { - continue; - } - - ResourceInformation entry = thisResources[i]; - ResourceInformation otherEntry = otherResources[i]; - if (entry.getName().equals(otherEntry.getName())) { - diff = entry.compareTo(otherEntry); - if (diff != 0) { - break; - } - } - } - } + public int hashCode() { + final int prime = 263167; + int result = 939769357; + for (ResourceInformation entry : getResources()) { + result = prime * result + entry.hashCode(); } - return Long.compare(diff, 0); + return result; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 3ab7ccd..2a04094 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.api.records; +import org.apache.curator.shaded.com.google.common.reflect.ClassPath; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.util.UnitsConversionUtil; @@ -34,8 +36,8 @@ private long minimumAllocation; private long maximumAllocation; - private static final String MEMORY_URI = "memory-mb"; - private static final String VCORES_URI = "vcores"; + public static final String MEMORY_URI = "memory-mb"; + public static final String VCORES_URI = "vcores"; public static final ResourceInformation MEMORY_MB = ResourceInformation.newInstance(MEMORY_URI, "Mi"); @@ -84,6 +86,16 @@ public void setUnits(String rUnits) { } /** + * Checking if a unit included by KNOWN_UNITS is an expensive operation. This + * can be avoided in critical path in RM. + * @param rUnits units for the resource + */ + @InterfaceAudience.Private + public void setUnitsWithoutValidation(String rUnits) { + this.units = rUnits; + } + + /** * Get the resource type. * * @return the resource type diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java index b5cc4d6..2cbbb44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java @@ -18,13 +18,18 @@ package org.apache.hadoop.yarn.api.records.impl; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import java.util.Arrays; +import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_MB; + /** *

* BaseResource extends Resource to handle base resources such @@ -54,48 +59,46 @@ * * @see Resource */ -@Public +@InterfaceAudience.Private @Unstable public class BaseResource extends Resource { private ResourceInformation memoryResInfo; private ResourceInformation vcoresResInfo; - protected ResourceInformation[] resources = null; - protected ResourceInformation[] readOnlyResources = null; - - // Number of mandatory resources, this is added to avoid invoke - // MandatoryResources.values().length, since values() internally will - // copy array, etc. - private static final int NUM_MANDATORY_RESOURCES = 2; - - protected enum MandatoryResources { - MEMORY(0), VCORES(1); - - private final int id; - - MandatoryResources(int id) { - this.id = id; - } - - public int getId() { - return this.id; - } - } public BaseResource() { // Base constructor. } public BaseResource(long memory, long vcores) { - this.memoryResInfo = ResourceInformation.newInstance(MEMORY, - ResourceInformation.MEMORY_MB.getUnits(), memory); - this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores); + this.memoryResInfo = BaseResource.newDefaultMemoryInformation(memory); + this.vcoresResInfo = BaseResource.newDefaultVCoresInformation(vcores); resources = new ResourceInformation[NUM_MANDATORY_RESOURCES]; - readOnlyResources = new ResourceInformation[NUM_MANDATORY_RESOURCES]; - resources[MandatoryResources.MEMORY.id] = memoryResInfo; - resources[MandatoryResources.VCORES.id] = vcoresResInfo; - readOnlyResources = Arrays.copyOf(resources, resources.length); + resources[MEMORY_INDEX] = memoryResInfo; + resources[VCORES_INDEX] = vcoresResInfo; + } + + public static ResourceInformation newDefaultMemoryInformation(long value) { + ResourceInformation ri = new ResourceInformation(); + ri.setName(ResourceInformation.MEMORY_URI); + ri.setValue(value); + ri.setResourceType(ResourceTypes.COUNTABLE); + ri.setUnitsWithoutValidation(MEMORY_MB.getUnits()); + ri.setMinimumAllocation(0); + ri.setMaximumAllocation(Long.MAX_VALUE); + return ri; + } + + public static ResourceInformation newDefaultVCoresInformation(long value) { + ResourceInformation ri = new ResourceInformation(); + ri.setName(ResourceInformation.VCORES_URI); + ri.setValue(value); + ri.setResourceType(ResourceTypes.COUNTABLE); + ri.setUnitsWithoutValidation(""); + ri.setMinimumAllocation(0); + ri.setMaximumAllocation(Long.MAX_VALUE); + return ri; } @Override @@ -131,7 +134,53 @@ public void setVirtualCores(int vcores) { } @Override - public ResourceInformation[] getResources() { - return readOnlyResources; + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof Resource)) { + return false; + } + Resource other = (Resource) obj; + if (getMemorySize() != other.getMemorySize() + || getVirtualCores() != other.getVirtualCores()) { + return false; + } + + return true; + } + + @Override + public int compareTo(Resource other) { + // compare memory and vcores first(in that order) to preserve + // existing behaviour + long diff = this.getMemorySize() - other.getMemorySize(); + if (diff == 0) { + return this.getVirtualCores() - other.getVirtualCores(); + } + return (int) Math.signum(diff); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(""); + return sb.toString(); + } + + @Override + public int hashCode() { + final int prime = 263167; + + // prime * result = 939769357 initially + int result = (int) (939769357 + getMemorySize()); + result = prime * result + getVirtualCores(); + + return result; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index ff22dd0..8530782 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -40,7 +40,7 @@ @Private @Unstable -public class ResourcePBImpl extends BaseResource { +public class ResourcePBImpl extends Resource { private static final Log LOG = LogFactory.getLog(ResourcePBImpl.class); @@ -95,7 +95,7 @@ public int getMemory() { @Override public long getMemorySize() { // memory should always be present - ResourceInformation ri = resources[MandatoryResources.MEMORY.getId()]; + ResourceInformation ri = resources[MEMORY_INDEX]; if (ri.getUnits().isEmpty()) { return ri.getValue(); @@ -119,7 +119,7 @@ public void setMemorySize(long memory) { @Override public int getVirtualCores() { // vcores should always be present - return (int) resources[MandatoryResources.VCORES.getId()].getValue(); + return (int) resources[VCORES_INDEX].getValue(); } @Override @@ -156,7 +156,6 @@ private void initResources() { resources[index].setValue(value); } } - readOnlyResources = Arrays.copyOf(resources, resources.length); this.setMemorySize(p.getMemory()); this.setVirtualCores(p.getVirtualCores()); } @@ -212,7 +211,6 @@ private void initResourcesMap() { } resources = new ResourceInformation[types.length]; - readOnlyResources = new ResourceInformation[types.length]; for (ResourceInformation entry : types) { int index = ResourceUtils.getResourceTypeIndex().get(entry.getName()); resources[index] = ResourceInformation.newInstance(entry); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index ffd4fec..6a4dd3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -73,7 +73,7 @@ private int compare(Resource lhs, Resource rhs) { boolean rhsGreater = false; int ret = 0; - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation lhsResourceInformation = lhs .getResourceInformation(i); @@ -111,10 +111,12 @@ public int compare(Resource clusterResource, Resource lhs, Resource rhs, // resources and then look for which resource has the biggest // share overall. ResourceInformation[] clusterRes = clusterResource.getResources(); + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + // If array creation shows up as a time sink, these arrays could be cached // because they're always the same length. - double[] lhsShares = new double[clusterRes.length]; - double[] rhsShares = new double[clusterRes.length]; + double[] lhsShares = new double[maxLength]; + double[] rhsShares = new double[maxLength]; double diff; try { @@ -124,7 +126,7 @@ public int compare(Resource clusterResource, Resource lhs, Resource rhs, calculateShares(clusterRes, lhs, rhs, lhsShares, rhsShares, max); diff = max[0] - max[1]; - } else if (clusterRes.length == 2) { + } else if (maxLength == 2) { // Special case to handle the common scenario of only CPU and memory // so that we can optimize for performance diff = calculateSharesForMandatoryResources(clusterRes, lhs, rhs, @@ -182,7 +184,8 @@ private void calculateShares(ResourceInformation[] clusterRes, Resource first, ResourceInformation[] firstRes = first.getResources(); ResourceInformation[] secondRes = second.getResources(); - for (int i = 0; i < clusterRes.length; i++) { + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { firstShares[i] = calculateShare(clusterRes[i], firstRes[i]); secondShares[i] = calculateShare(clusterRes[i], secondRes[i]); } @@ -215,7 +218,8 @@ private int calculateSharesForMandatoryResources( int firstSub = 0; int secondSub = 0; - for (int i = 0; i < clusterRes.length; i++) { + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { firstShares[i] = calculateShare(clusterRes[i], firstRes[i]); secondShares[i] = calculateShare(clusterRes[i], secondRes[i]); @@ -280,7 +284,8 @@ private void calculateShares(ResourceInformation[] clusterRes, Resource first, max[0] = 0.0; max[1] = 0.0; - for (int i = 0; i < clusterRes.length; i++) { + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { firstShares[i] = calculateShare(clusterRes[i], firstRes[i]); secondShares[i] = calculateShare(clusterRes[i], secondRes[i]); @@ -339,7 +344,7 @@ private double compareShares(double[] lhsShares, double[] rhsShares) { public long computeAvailableContainers(Resource available, Resource required) { long min = Long.MAX_VALUE; - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation availableResource = available .getResourceInformation(i); @@ -358,11 +363,12 @@ public long computeAvailableContainers(Resource available, @Override public float divide(Resource clusterResource, Resource numerator, Resource denominator) { + int nKnownResourceTypes = ResourceUtils.getNumberOfKnownResourceTypes(); ResourceInformation[] clusterRes = clusterResource.getResources(); // We have to provide the calculateShares() method with somewhere to store // the shares. We don't actually need these shares afterwards. - double[] numeratorShares = new double[clusterRes.length]; - double[] denominatorShares = new double[clusterRes.length]; + double[] numeratorShares = new double[nKnownResourceTypes]; + double[] denominatorShares = new double[nKnownResourceTypes]; // We also have to provide a place for calculateShares() to store the max // shares so that we can use them. double[] max = new double[2]; @@ -386,7 +392,7 @@ public boolean isInvalidDivisor(Resource r) { @Override public float ratio(Resource a, Resource b) { float ratio = 0.0f; - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation aResourceInformation = a.getResourceInformation(i); ResourceInformation bResourceInformation = b.getResourceInformation(i); @@ -407,7 +413,7 @@ public Resource divideAndCeil(Resource numerator, int denominator) { public Resource divideAndCeil(Resource numerator, long denominator) { Resource ret = Resource.newInstance(numerator); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation resourceInformation = ret.getResourceInformation(i); resourceInformation @@ -428,7 +434,7 @@ public Resource divideAndCeil(Resource numerator, float denominator) { public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { Resource ret = Resource.newInstance(r); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation minimumResourceInformation = minimumResource @@ -474,7 +480,7 @@ public Resource roundDown(Resource r, Resource stepFactor) { private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) { Resource ret = Resource.newInstance(r); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation stepFactorResourceInformation = stepFactor @@ -513,7 +519,7 @@ public Resource multiplyAndNormalizeDown(Resource r, double by, private Resource multiplyAndNormalize(Resource r, double by, Resource stepFactor, boolean roundUp) { Resource ret = Resource.newInstance(r); - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation stepFactorResourceInformation = stepFactor @@ -542,7 +548,7 @@ private Resource multiplyAndNormalize(Resource r, double by, @Override public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) { - int maxLength = ResourceUtils.getResourceTypesArray().length; + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation sResourceInformation = smaller .getResourceInformation(i); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 1e2ce15..bee94f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -24,12 +24,9 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; -import org.apache.hadoop.yarn.api.records.impl.BaseResource; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.util.UnitsConversionUtil; -import java.util.Arrays; - /** * Resources is a computation class which provides a set of apis to do * mathematical operations on Resource object. @@ -45,9 +42,11 @@ * Helper class to create a resource with a fixed value for all resource * types. For example, a NONE resource which returns 0 for any resource type. */ - static class FixedValueResource extends BaseResource { + @InterfaceAudience.Private + @Unstable + static class FixedValueResource extends Resource { - private long resourceValue; + private final long resourceValue; private String name; /** @@ -101,6 +100,19 @@ public void setVirtualCores(int virtualCores) { } @Override + public void setResourceInformation(int index, + ResourceInformation resourceInformation) + throws ResourceNotFoundException { + throw new RuntimeException(name + " cannot be modified!"); + } + + @Override + public void setResourceValue(int index, long value) + throws ResourceNotFoundException { + throw new RuntimeException(name + " cannot be modified!"); + } + + @Override public void setResourceInformation(String resource, ResourceInformation resourceInformation) throws ResourceNotFoundException { @@ -117,19 +129,16 @@ private void initResourceMap() { ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); if (types != null) { resources = new ResourceInformation[types.length]; - readOnlyResources = new ResourceInformation[types.length]; for (int index = 0; index < types.length; index++) { resources[index] = ResourceInformation.newInstance(types[index]); resources[index].setValue(resourceValue); - - // this is a fix for getVirtualCores returning an int - if (resourceValue > Integer.MAX_VALUE && ResourceInformation.VCORES - .getName().equals(resources[index].getName())) { - resources[index].setValue((long) Integer.MAX_VALUE); - } } } - readOnlyResources = Arrays.copyOf(resources, resources.length); + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj); } } -- 2.10.1 (Apple Git-78)