From e81407ab9b9c662d1e3e3256c6d5795016340595 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Fri, 27 Oct 2017 18:22:01 +0530 Subject: [PATCH] YARN-7410 --- .../apache/hadoop/yarn/api/records/Resource.java | 94 ++++++++++++++++++ .../hadoop/yarn/util/resource/ResourceUtils.java | 15 ++- .../hadoop/yarn/util/resource/Resources.java | 105 +-------------------- .../hadoop/yarn/util/resource/TestResources.java | 4 +- 4 files changed, 114 insertions(+), 104 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 9a5bc79ae08..eda4987112f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.records.impl.LightWeightResource; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; @@ -69,6 +70,99 @@ protected static final int MEMORY_INDEX = 0; protected static final int VCORES_INDEX = 1; + /** + * Helper class to create a resource with a fixed value for all resource + * types. For example, a NONE resource which returns 0 for any resource type. + */ + @InterfaceAudience.Public + @Unstable + public + static class FixedValueResource extends Resource { + + private final long resourceValue; + private String name; + + /** + * Constructor for a fixed value resource. + * @param rName the name of the resource + * @param value the fixed value to be returned for all resource types + */ + public FixedValueResource(String rName, long value) { + this.resourceValue = value; + this.name = rName; + initResourceMap(); + } + + @Override + @SuppressWarnings("deprecation") + public int getMemory() { + return castToIntSafely(resourceValue); + } + + @Override + public long getMemorySize() { + return this.resourceValue; + } + + @Override + @SuppressWarnings("deprecation") + public void setMemory(int memory) { + throw new RuntimeException(name + " cannot be modified!"); + } + + @Override + public void setMemorySize(long memory) { + throw new RuntimeException(name + " cannot be modified!"); + } + + @Override + public int getVirtualCores() { + return castToIntSafely(resourceValue); + } + + @Override + public void setVirtualCores(int virtualCores) { + throw new RuntimeException(name + " cannot be modified!"); + } + + @Override + public void setResourceInformation(int index, + ResourceInformation resourceInformation) + throws ResourceNotFoundException { + throw new RuntimeException(name + " cannot be modified!"); + } + + @Override + public void setResourceValue(int index, long value) + throws ResourceNotFoundException { + throw new RuntimeException(name + " cannot be modified!"); + } + + @Override + public void setResourceInformation(String resource, + ResourceInformation resourceInformation) + throws ResourceNotFoundException { + throw new RuntimeException(name + " cannot be modified!"); + } + + @Override + public void setResourceValue(String resource, long value) + throws ResourceNotFoundException { + throw new RuntimeException(name + " cannot be modified!"); + } + + private void initResourceMap() { + ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); + if (types != null) { + resources = new ResourceInformation[types.length]; + for (int index = 0; index < types.length; index++) { + resources[index] = ResourceInformation.newInstance(types[index]); + resources[index].setValue(resourceValue); + } + } + } + } + @Public @Stable public static Resource newInstance(int memory, int vCores) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 3a09de5d6ba..4912ea48cec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -68,6 +68,13 @@ static final Log LOG = LogFactory.getLog(ResourceUtils.class); + static Resource UNBOUNDED; + static Resource NONE; + + static { + reInitializeStaticResources(); + } + private ResourceUtils() { } @@ -259,9 +266,16 @@ static void initializeResourcesMap(Configuration conf) { public static void initializeResourcesFromResourceInformationMap( Map resourceInformationMap) { resourceTypes = Collections.unmodifiableMap(resourceInformationMap); + numKnownResourceTypes = resourceTypes.size(); updateKnownResources(); updateResourceTypeIndex(); initializedResources = true; + reInitializeStaticResources(); + } + + private static void reInitializeStaticResources() { + UNBOUNDED = new Resource.FixedValueResource("UNBOUNDED", Long.MAX_VALUE); + NONE = new Resource.FixedValueResource("NONE", 0L); } private static void updateKnownResources() { @@ -350,7 +364,6 @@ private static void initializeResourceTypesIfNeeded(Configuration conf, } } } - numKnownResourceTypes = resourceTypes.size(); } private static Map getResourceTypes( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 36909461951..40f77844c54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -38,98 +38,6 @@ private static final Log LOG = LogFactory.getLog(Resources.class); - /** - * Helper class to create a resource with a fixed value for all resource - * types. For example, a NONE resource which returns 0 for any resource type. - */ - @InterfaceAudience.Private - @Unstable - static class FixedValueResource extends Resource { - - private final long resourceValue; - private String name; - - /** - * Constructor for a fixed value resource. - * @param rName the name of the resource - * @param value the fixed value to be returned for all resource types - */ - FixedValueResource(String rName, long value) { - this.resourceValue = value; - this.name = rName; - initResourceMap(); - } - - @Override - @SuppressWarnings("deprecation") - public int getMemory() { - return castToIntSafely(resourceValue); - } - - @Override - public long getMemorySize() { - return this.resourceValue; - } - - @Override - @SuppressWarnings("deprecation") - public void setMemory(int memory) { - throw new RuntimeException(name + " cannot be modified!"); - } - - @Override - public void setMemorySize(long memory) { - throw new RuntimeException(name + " cannot be modified!"); - } - - @Override - public int getVirtualCores() { - return castToIntSafely(resourceValue); - } - - @Override - public void setVirtualCores(int virtualCores) { - throw new RuntimeException(name + " cannot be modified!"); - } - - @Override - public void setResourceInformation(int index, - ResourceInformation resourceInformation) - throws ResourceNotFoundException { - throw new RuntimeException(name + " cannot be modified!"); - } - - @Override - public void setResourceValue(int index, long value) - throws ResourceNotFoundException { - throw new RuntimeException(name + " cannot be modified!"); - } - - @Override - public void setResourceInformation(String resource, - ResourceInformation resourceInformation) - throws ResourceNotFoundException { - throw new RuntimeException(name + " cannot be modified!"); - } - - @Override - public void setResourceValue(String resource, long value) - throws ResourceNotFoundException { - throw new RuntimeException(name + " cannot be modified!"); - } - - private void initResourceMap() { - ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); - if (types != null) { - resources = new ResourceInformation[types.length]; - for (int index = 0; index < types.length; index++) { - resources[index] = ResourceInformation.newInstance(types[index]); - resources[index].setValue(resourceValue); - } - } - } - } - public static Resource createResource(int memory) { return createResource(memory, (memory > 0) ? 1 : 0); } @@ -138,11 +46,6 @@ public static Resource createResource(int memory, int cores) { return Resource.newInstance(memory, cores); } - private static final Resource UNBOUNDED = - new FixedValueResource("UNBOUNDED", Long.MAX_VALUE); - - private static final Resource NONE = new FixedValueResource("NONE", 0L); - public static Resource createResource(long memory) { return createResource(memory, (memory > 0) ? 1 : 0); } @@ -152,7 +55,7 @@ public static Resource createResource(long memory, int cores) { } public static Resource none() { - return NONE; + return ResourceUtils.NONE; } /** @@ -162,11 +65,11 @@ public static Resource none() { * {@code false} otherwise */ public static boolean isNone(Resource other) { - return NONE.equals(other); + return ResourceUtils.NONE.equals(other); } public static Resource unbounded() { - return UNBOUNDED; + return ResourceUtils.UNBOUNDED; } public static Resource clone(Resource res) { @@ -240,7 +143,7 @@ public static Resource subtractFromNonNegative(Resource lhs, Resource rhs) { } public static Resource negate(Resource resource) { - return subtract(NONE, resource); + return subtract(ResourceUtils.NONE, resource); } public static Resource multiplyTo(Resource lhs, double by) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index a8404fbaee7..30db30a0c96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -44,11 +44,11 @@ static class ExtendedResources extends Resources { public static Resource unbounded() { - return new FixedValueResource("UNBOUNDED", Long.MAX_VALUE); + return new Resource.FixedValueResource("UNBOUNDED", Long.MAX_VALUE); } public static Resource none() { - return new FixedValueResource("NONE", 0L); + return new Resource.FixedValueResource("NONE", 0L); } } -- 2.13.5 (Apple Git-94)