From ce0cfdd43168afb1c9818a142e734b6721581ac3 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Tue, 7 Nov 2017 12:34:58 +0530 Subject: [PATCH] YARN-6909 --- .../apache/hadoop/yarn/api/records/Resource.java | 46 +++++++++----- .../yarn/api/records/impl/LightWeightResource.java | 58 ++++++++++------- .../yarn/api/records/impl/pb/ResourcePBImpl.java | 73 +++++++++------------- .../scheduler/ClusterNodeTracker.java | 2 +- 4 files changed, 97 insertions(+), 82 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index be292ff9a20..52d95945325 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -28,9 +28,9 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.impl.LightWeightResource; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; -import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.ResourceUtils; /** @@ -75,34 +75,27 @@ @Public @Stable public static Resource newInstance(int memory, int vCores) { - if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { - Resource ret = Records.newRecord(Resource.class); - ret.setMemorySize(memory); - ret.setVirtualCores(vCores); - return ret; - } return new LightWeightResource(memory, vCores); } @Public @Stable public static Resource newInstance(long memory, int vCores) { - if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { - Resource ret = Records.newRecord(Resource.class); - ret.setMemorySize(memory); - ret.setVirtualCores(vCores); - return ret; - } return new LightWeightResource(memory, vCores); } @InterfaceAudience.Private @InterfaceStability.Unstable public static Resource newInstance(Resource resource) { - Resource ret = Resource.newInstance(resource.getMemorySize(), - resource.getVirtualCores()); - if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { - Resource.copy(resource, ret); + Resource ret; + int numberOfKnownResourceTypes = ResourceUtils + .getNumberOfKnownResourceTypes(); + if (numberOfKnownResourceTypes > 2) { + ret = new LightWeightResource(resource.getMemorySize(), + resource.getVirtualCores(), resource.getResources()); + } else { + ret = new LightWeightResource(resource.getMemorySize(), + resource.getVirtualCores()); } return ret; } @@ -483,4 +476,23 @@ protected static int castToIntSafely(long value) { } return Long.valueOf(value).intValue(); } + + /** + * Create ResourceInformation with basic fields. + * @param name Resource Type Name + * @param unit Default unit of provided resource type + * @param value Value associated with giveb resource + * @return ResourceInformation object + */ + protected static ResourceInformation newDefaultInformation(String name, + String unit, long value) { + ResourceInformation ri = new ResourceInformation(); + ri.setName(name); + ri.setValue(value); + ri.setResourceType(ResourceTypes.COUNTABLE); + ri.setUnitsWithoutValidation(unit); + ri.setMinimumAllocation(0); + ri.setMaximumAllocation(Long.MAX_VALUE); + return ri; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java index 7b07bbd9491..c36166b2764 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java @@ -23,14 +23,13 @@ import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; -import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_MB; -import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_URI; -import static org.apache.hadoop.yarn.api.records.ResourceInformation.VCORES_URI; +import static org.apache.hadoop.yarn.api.records.ResourceInformation.*; /** *

- * LightResource extends Resource to handle base resources such + * LightWeightResource extends Resource to handle base resources such * as memory and CPU. * TODO: We have a long term plan to use AbstractResource when additional * resource types are to be handled as well. @@ -66,27 +65,42 @@ private ResourceInformation memoryResInfo; private ResourceInformation vcoresResInfo; - public LightWeightResource(long memory, long vcores) { - this.memoryResInfo = LightWeightResource.newDefaultInformation(MEMORY_URI, - MEMORY_MB.getUnits(), memory); - this.vcoresResInfo = LightWeightResource.newDefaultInformation(VCORES_URI, - "", vcores); + public LightWeightResource(long memory, int vcores) { + int numberOfKnownResourceTypes = ResourceUtils + .getNumberOfKnownResourceTypes(); + initResourceInformations(memory, vcores, numberOfKnownResourceTypes); + + if (numberOfKnownResourceTypes > 2) { + ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); + for (int i = 2; i < numberOfKnownResourceTypes; i++) { + resources[i] = new ResourceInformation(); + ResourceInformation.copy(types[i], resources[i]); + } + } + } - resources = new ResourceInformation[NUM_MANDATORY_RESOURCES]; - resources[MEMORY_INDEX] = memoryResInfo; - resources[VCORES_INDEX] = vcoresResInfo; + public LightWeightResource(long memory, int vcores, + ResourceInformation[] source) { + int numberOfKnownResourceTypes = ResourceUtils + .getNumberOfKnownResourceTypes(); + initResourceInformations(memory, vcores, numberOfKnownResourceTypes); + + for (int i = 2; i < numberOfKnownResourceTypes; i++) { + resources[i] = new ResourceInformation(); + ResourceInformation.copy(source[i], resources[i]); + } } - private static ResourceInformation newDefaultInformation(String name, - String unit, long value) { - ResourceInformation ri = new ResourceInformation(); - ri.setName(name); - ri.setValue(value); - ri.setResourceType(ResourceTypes.COUNTABLE); - ri.setUnitsWithoutValidation(unit); - ri.setMinimumAllocation(0); - ri.setMaximumAllocation(Long.MAX_VALUE); - return ri; + private void initResourceInformations(long memory, int vcores, + int numberOfKnownResourceTypes) { + this.memoryResInfo = newDefaultInformation(MEMORY_URI, MEMORY_MB.getUnits(), + memory); + this.vcoresResInfo = newDefaultInformation(VCORES_URI, VCORES.getUnits(), + vcores); + + resources = new ResourceInformation[numberOfKnownResourceTypes]; + resources[MEMORY_INDEX] = memoryResInfo; + resources[VCORES_INDEX] = vcoresResInfo; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 4ae64c2582a..6afcebf5a82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -35,7 +35,6 @@ import java.util.Map; - @Private @Unstable public class ResourcePBImpl extends Resource { @@ -50,7 +49,7 @@ static ResourceProto getProto(Resource r) { final ResourcePBImpl pb; if (r instanceof ResourcePBImpl) { - pb = (ResourcePBImpl)r; + pb = (ResourcePBImpl) r; } else { pb = new ResourcePBImpl(); pb.setMemorySize(r.getMemorySize()); @@ -111,7 +110,7 @@ public void setMemory(int memory) { @Override public void setMemorySize(long memory) { maybeInitBuilder(); - getResourceInformation(ResourceInformation.MEMORY_URI).setValue(memory); + resources[MEMORY_INDEX].setValue(memory); } @Override @@ -123,7 +122,7 @@ public int getVirtualCores() { @Override public void setVirtualCores(int vCores) { maybeInitBuilder(); - getResourceInformation(ResourceInformation.VCORES_URI).setValue(vCores); + resources[VCORES_INDEX].setValue(vCores); } private void initResources() { @@ -131,33 +130,41 @@ private void initResources() { return; } ResourceProtoOrBuilder p = viaProto ? proto : builder; - initResourcesMap(); + ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); Map indexMap = ResourceUtils.getResourceTypeIndex(); + resources = new ResourceInformation[types.length]; + for (ResourceInformationProto entry : p.getResourceValueMapList()) { - ResourceTypes type = - entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) : - ResourceTypes.COUNTABLE; - - // When unit not specified in proto, use the default unit. - String units = - entry.hasUnits() ? entry.getUnits() : ResourceUtils.getDefaultUnit( - entry.getKey()); - long value = entry.hasValue() ? entry.getValue() : 0L; - ResourceInformation ri = ResourceInformation - .newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE); Integer index = indexMap.get(entry.getKey()); if (index == null) { - LOG.warn("Got unknown resource type: " + ri.getName() + "; skipping"); + LOG.warn("Got unknown resource type: " + entry.getKey() + "; skipping"); } else { - resources[index].setResourceType(ri.getResourceType()); - resources[index].setUnits(ri.getUnits()); - resources[index].setValue(value); + resources[index] = newDefaultInformation(types[index], entry); } } + resources[MEMORY_INDEX] = + ResourceInformation.newInstance(ResourceInformation.MEMORY_MB); + resources[VCORES_INDEX] = + ResourceInformation.newInstance(ResourceInformation.VCORES); this.setMemorySize(p.getMemory()); this.setVirtualCores(p.getVirtualCores()); } + private static ResourceInformation newDefaultInformation( + ResourceInformation resourceInformation, ResourceInformationProto entry) { + ResourceInformation ri = new ResourceInformation(); + ri.setName(resourceInformation.getName()); + ri.setMinimumAllocation(resourceInformation.getMinimumAllocation()); + ri.setMaximumAllocation(resourceInformation.getMaximumAllocation()); + ri.setResourceType(entry.hasType() + ? ProtoUtils.convertFromProtoFormat(entry.getType()) + : ResourceTypes.COUNTABLE); + ri.setUnits( + entry.hasUnits() ? entry.getUnits() : resourceInformation.getUnits()); + ri.setValue(entry.hasValue() ? entry.getValue() : 0L); + return ri; + } + @Override public void setResourceInformation(String resource, ResourceInformation resourceInformation) { @@ -166,10 +173,8 @@ public void setResourceInformation(String resource, throw new IllegalArgumentException( "resource and/or resourceInformation cannot be null"); } - if (!resource.equals(resourceInformation.getName())) { - resourceInformation.setName(resource); - } - ResourceInformation storedResourceInfo = getResourceInformation(resource); + ResourceInformation storedResourceInfo = super.getResourceInformation( + resource); ResourceInformation.copy(resourceInformation, storedResourceInfo); } @@ -195,25 +200,9 @@ public long getResourceValue(String resource) return super.getResourceValue(resource); } - private void initResourcesMap() { - if (resources == null) { - ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); - if (types == null) { - throw new YarnRuntimeException( - "Got null return value from ResourceUtils.getResourceTypes()"); - } - - resources = new ResourceInformation[types.length]; - for (ResourceInformation entry : types) { - int index = ResourceUtils.getResourceTypeIndex().get(entry.getName()); - resources[index] = ResourceInformation.newInstance(entry); - } - } - } - synchronized private void mergeLocalToBuilder() { builder.clearResourceValueMap(); - if(resources != null && resources.length != 0) { + if (resources != null && resources.length != 0) { for (ResourceInformation resInfo : resources) { ResourceInformationProto.Builder e = ResourceInformationProto .newBuilder(); @@ -236,4 +225,4 @@ private void mergeLocalToProto() { proto = builder.build(); viaProto = true; } -} +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java index ccec6bc6a89..86802c6b1b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java @@ -55,7 +55,7 @@ private Map nodeNameToNodeMap = new HashMap<>(); private Map> nodesPerRack = new HashMap<>(); - private Resource clusterCapacity = Resources.clone(Resources.none()); + private Resource clusterCapacity = Resources.createResource(0, 0); private Resource staleClusterCapacity = null; // Max allocation -- 2.13.6 (Apple Git-96)