From 75cb92fb184b49e8379bcf69a23bbc4fa541cf09 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Fri, 3 Nov 2017 13:44:07 +0530 Subject: [PATCH] YARN-6909 --- .../apache/hadoop/yarn/api/records/Resource.java | 47 ++++++++------ .../yarn/api/records/impl/LightWeightResource.java | 45 +++++++++----- .../yarn/api/records/impl/pb/ResourcePBImpl.java | 71 ++++++++-------------- .../scheduler/ClusterNodeTracker.java | 2 +- 4 files changed, 87 insertions(+), 78 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index be292ff9a20..a7bfbbfeb13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -28,9 +28,9 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.impl.LightWeightResource; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; -import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.ResourceUtils; /** @@ -75,35 +75,29 @@ @Public @Stable public static Resource newInstance(int memory, int vCores) { - if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { - Resource ret = Records.newRecord(Resource.class); - ret.setMemorySize(memory); - ret.setVirtualCores(vCores); - return ret; - } return new LightWeightResource(memory, vCores); } @Public @Stable public static Resource newInstance(long memory, int vCores) { - if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { - Resource ret = Records.newRecord(Resource.class); - ret.setMemorySize(memory); - ret.setVirtualCores(vCores); - return ret; - } return new LightWeightResource(memory, vCores); } @InterfaceAudience.Private @InterfaceStability.Unstable public static Resource newInstance(Resource resource) { - Resource ret = Resource.newInstance(resource.getMemorySize(), - resource.getVirtualCores()); - if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { - Resource.copy(resource, ret); + Resource ret; + int numberOfKnownResourceTypes = ResourceUtils + .getNumberOfKnownResourceTypes(); + if (numberOfKnownResourceTypes > 2) { + ret = new LightWeightResource(resource.getMemorySize(), + resource.getVirtualCores(), resource.getResources(), + numberOfKnownResourceTypes); + return ret; } + ret = new LightWeightResource(resource.getMemorySize(), + resource.getVirtualCores()); return ret; } @@ -483,4 +477,23 @@ protected static int castToIntSafely(long value) { } return Long.valueOf(value).intValue(); } + + /** + * Create ResourceInformation with basic fields. + * @param name Resource Type Name + * @param unit Default unit of provided resource type + * @param value Value associated with giveb resource + * @return ResourceInformation object + */ + protected static ResourceInformation newDefaultInformation(String name, + String unit, long value) { + ResourceInformation ri = new ResourceInformation(); + ri.setName(name); + ri.setValue(value); + ri.setResourceType(ResourceTypes.COUNTABLE); + ri.setUnitsWithoutValidation(unit); + ri.setMinimumAllocation(0); + ri.setMaximumAllocation(Long.MAX_VALUE); + return ri; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java index 7b07bbd9491..9cd266b1241 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_MB; import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_URI; @@ -30,7 +31,7 @@ /** *

- * LightResource extends Resource to handle base resources such + * LightWeightResource extends Resource to handle base resources such * as memory and CPU. * TODO: We have a long term plan to use AbstractResource when additional * resource types are to be handled as well. @@ -67,26 +68,38 @@ private ResourceInformation vcoresResInfo; public LightWeightResource(long memory, long vcores) { - this.memoryResInfo = LightWeightResource.newDefaultInformation(MEMORY_URI, - MEMORY_MB.getUnits(), memory); - this.vcoresResInfo = LightWeightResource.newDefaultInformation(VCORES_URI, - "", vcores); + int numberOfKnownResourceTypes = ResourceUtils + .getNumberOfKnownResourceTypes(); + this.memoryResInfo = newDefaultInformation(MEMORY_URI, MEMORY_MB.getUnits(), + memory); + this.vcoresResInfo = newDefaultInformation(VCORES_URI, "", vcores); - resources = new ResourceInformation[NUM_MANDATORY_RESOURCES]; + resources = new ResourceInformation[numberOfKnownResourceTypes]; resources[MEMORY_INDEX] = memoryResInfo; resources[VCORES_INDEX] = vcoresResInfo; + + if (numberOfKnownResourceTypes > 2) { + ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); + for (int i = 2; i < numberOfKnownResourceTypes; i++) { + resources[i] = new ResourceInformation(); + ResourceInformation.copy(types[i], resources[i]); + } + } } - private static ResourceInformation newDefaultInformation(String name, - String unit, long value) { - ResourceInformation ri = new ResourceInformation(); - ri.setName(name); - ri.setValue(value); - ri.setResourceType(ResourceTypes.COUNTABLE); - ri.setUnitsWithoutValidation(unit); - ri.setMinimumAllocation(0); - ri.setMaximumAllocation(Long.MAX_VALUE); - return ri; + public LightWeightResource(long memory, int vcores, + ResourceInformation[] source, int numberOfKnownResourceTypes) { + this.memoryResInfo = newDefaultInformation(MEMORY_URI, MEMORY_MB.getUnits(), + memory); + this.vcoresResInfo = newDefaultInformation(VCORES_URI, "", vcores); + + resources = new ResourceInformation[numberOfKnownResourceTypes]; + resources[MEMORY_INDEX] = memoryResInfo; + resources[VCORES_INDEX] = vcoresResInfo; + for (int i = 2; i < numberOfKnownResourceTypes; i++) { + resources[i] = new ResourceInformation(); + ResourceInformation.copy(source[i], resources[i]); + } } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 4ae64c2582a..6fbfcf83884 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -35,7 +35,6 @@ import java.util.Map; - @Private @Unstable public class ResourcePBImpl extends Resource { @@ -50,7 +49,7 @@ static ResourceProto getProto(Resource r) { final ResourcePBImpl pb; if (r instanceof ResourcePBImpl) { - pb = (ResourcePBImpl)r; + pb = (ResourcePBImpl) r; } else { pb = new ResourcePBImpl(); pb.setMemorySize(r.getMemorySize()); @@ -111,7 +110,7 @@ public void setMemory(int memory) { @Override public void setMemorySize(long memory) { maybeInitBuilder(); - getResourceInformation(ResourceInformation.MEMORY_URI).setValue(memory); + resources[MEMORY_INDEX].setValue(memory); } @Override @@ -123,7 +122,7 @@ public int getVirtualCores() { @Override public void setVirtualCores(int vCores) { maybeInitBuilder(); - getResourceInformation(ResourceInformation.VCORES_URI).setValue(vCores); + resources[VCORES_INDEX].setValue(vCores); } private void initResources() { @@ -131,31 +130,33 @@ private void initResources() { return; } ResourceProtoOrBuilder p = viaProto ? proto : builder; - initResourcesMap(); + ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); Map indexMap = ResourceUtils.getResourceTypeIndex(); + resources = new ResourceInformation[types.length]; + for (ResourceInformationProto entry : p.getResourceValueMapList()) { - ResourceTypes type = - entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) : - ResourceTypes.COUNTABLE; - - // When unit not specified in proto, use the default unit. - String units = - entry.hasUnits() ? entry.getUnits() : ResourceUtils.getDefaultUnit( - entry.getKey()); - long value = entry.hasValue() ? entry.getValue() : 0L; - ResourceInformation ri = ResourceInformation - .newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE); Integer index = indexMap.get(entry.getKey()); if (index == null) { - LOG.warn("Got unknown resource type: " + ri.getName() + "; skipping"); + LOG.warn("Got unknown resource type: " + entry.getKey() + "; skipping"); } else { - resources[index].setResourceType(ri.getResourceType()); - resources[index].setUnits(ri.getUnits()); - resources[index].setValue(value); + resources[index] = newDefaultInformation(types[index], entry); } } - this.setMemorySize(p.getMemory()); - this.setVirtualCores(p.getVirtualCores()); + } + + private static ResourceInformation newDefaultInformation( + ResourceInformation resourceInformation, ResourceInformationProto entry) { + ResourceInformation ri = new ResourceInformation(); + ri.setName(resourceInformation.getName()); + ri.setMinimumAllocation(resourceInformation.getMinimumAllocation()); + ri.setMaximumAllocation(resourceInformation.getMaximumAllocation()); + ri.setResourceType(entry.hasType() + ? ProtoUtils.convertFromProtoFormat(entry.getType()) + : ResourceTypes.COUNTABLE); + ri.setUnits( + entry.hasUnits() ? entry.getUnits() : resourceInformation.getUnits()); + ri.setValue(entry.hasValue() ? entry.getValue() : 0L); + return ri; } @Override @@ -166,10 +167,8 @@ public void setResourceInformation(String resource, throw new IllegalArgumentException( "resource and/or resourceInformation cannot be null"); } - if (!resource.equals(resourceInformation.getName())) { - resourceInformation.setName(resource); - } - ResourceInformation storedResourceInfo = getResourceInformation(resource); + ResourceInformation storedResourceInfo = super.getResourceInformation( + resource); ResourceInformation.copy(resourceInformation, storedResourceInfo); } @@ -195,25 +194,9 @@ public long getResourceValue(String resource) return super.getResourceValue(resource); } - private void initResourcesMap() { - if (resources == null) { - ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); - if (types == null) { - throw new YarnRuntimeException( - "Got null return value from ResourceUtils.getResourceTypes()"); - } - - resources = new ResourceInformation[types.length]; - for (ResourceInformation entry : types) { - int index = ResourceUtils.getResourceTypeIndex().get(entry.getName()); - resources[index] = ResourceInformation.newInstance(entry); - } - } - } - synchronized private void mergeLocalToBuilder() { builder.clearResourceValueMap(); - if(resources != null && resources.length != 0) { + if (resources != null && resources.length != 0) { for (ResourceInformation resInfo : resources) { ResourceInformationProto.Builder e = ResourceInformationProto .newBuilder(); @@ -236,4 +219,4 @@ private void mergeLocalToProto() { proto = builder.build(); viaProto = true; } -} +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java index ccec6bc6a89..86802c6b1b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java @@ -55,7 +55,7 @@ private Map nodeNameToNodeMap = new HashMap<>(); private Map> nodesPerRack = new HashMap<>(); - private Resource clusterCapacity = Resources.clone(Resources.none()); + private Resource clusterCapacity = Resources.createResource(0, 0); private Resource staleClusterCapacity = null; // Max allocation -- 2.13.5 (Apple Git-94)