From 9f8a506ed1455b56b29a85974fcc12cedaf6aa31 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Wed, 12 Apr 2017 20:32:36 +0530 Subject: [PATCH] YARN-6471 --- .../java/org/apache/hadoop/util/StringUtils.java | 31 +++ .../hadoop/yarn/util/UnitsConversionUtil.java | 218 +++++++++++++++++++ .../scheduler/capacity/AbstractCSQueue.java | 58 +++++ .../scheduler/capacity/CSQueue.java | 15 ++ .../capacity/CapacitySchedulerConfiguration.java | 63 ++++++ .../scheduler/capacity/ParentQueue.java | 40 +++- .../scheduler/capacity/QueueCapacities.java | 2 +- .../scheduler/capacity/ResourceInformation.java | 235 +++++++++++++++++++++ 8 files changed, 659 insertions(+), 3 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ResourceInformation.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java index e773806..132fad9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java @@ -1142,4 +1142,35 @@ public static boolean equalsIgnoreCase(String s1, String s2) { return s1.equalsIgnoreCase(s2); } + /** + *

Checks if the String contains only unicode letters.

+ * + *

null will return false. + * An empty String (length()=0) will return true.

+ * + *
+   * StringUtils.isAlpha(null)   = false
+   * StringUtils.isAlpha("")     = true
+   * StringUtils.isAlpha("  ")   = false
+   * StringUtils.isAlpha("abc")  = true
+   * StringUtils.isAlpha("ab2c") = false
+   * StringUtils.isAlpha("ab-c") = false
+   * 
+ * + * @param str the String to check, may be null + * @return true if only contains letters, and is non-null + */ + public static boolean isAlpha(String str) { + if (str == null) { + return false; + } + int sz = str.length(); + for (int i = 0; i < sz; i++) { + if (Character.isLetter(str.charAt(i)) == false) { + return false; + } + } + return true; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java new file mode 100644 index 0000000..9455dc9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.util; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import java.math.BigInteger; +import java.util.*; + +/** + * A util to convert values in one unit to another. Units refers to whether + * the value is expressed in pico, nano, etc. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class UnitsConversionUtil { + + /** + * Helper class for encapsulating conversion values. + */ + public static class Converter { + private long numerator; + private long denominator; + + Converter(long n, long d) { + this.numerator = n; + this.denominator = d; + } + } + + private static final String[] UNITS = + { "p", "n", "u", "m", "", "k", "M", "G", "T", "P", "Ki", "Mi", "Gi", "Ti", + "Pi" }; + private static final List SORTED_UNITS = Arrays.asList(UNITS); + public static final Set KNOWN_UNITS = createKnownUnitsSet(); + private static final Converter PICO = + new Converter(1L, 1000L * 1000L * 1000L * 1000L); + private static final Converter NANO = + new Converter(1L, 1000L * 1000L * 1000L); + private static final Converter MICRO = new Converter(1L, 1000L * 1000L); + private static final Converter MILLI = new Converter(1L, 1000L); + private static final Converter BASE = new Converter(1L, 1L); + private static final Converter KILO = new Converter(1000L, 1L); + private static final Converter MEGA = new Converter(1000L * 1000L, 1L); + private static final Converter GIGA = + new Converter(1000L * 1000L * 1000L, 1L); + private static final Converter TERA = + new Converter(1000L * 1000L * 1000L * 1000L, 1L); + private static final Converter PETA = + new Converter(1000L * 1000L * 1000L * 1000L * 1000L, 1L); + + private static final Converter KILO_BINARY = new Converter(1024L, 1L); + private static final Converter MEGA_BINARY = new Converter(1024L * 1024L, 1L); + private static final Converter GIGA_BINARY = + new Converter(1024L * 1024L * 1024L, 1L); + private static final Converter TERA_BINARY = + new Converter(1024L * 1024L * 1024L * 1024L, 1L); + private static final Converter PETA_BINARY = + new Converter(1024L * 1024L * 1024L * 1024L * 1024L, 1L); + + private static Set createKnownUnitsSet() { + Set ret = new HashSet<>(); + ret.addAll(Arrays.asList(UNITS)); + return ret; + } + + private static Converter getConverter(String unit) { + switch (unit) { + case "p": + return PICO; + case "n": + return NANO; + case "u": + return MICRO; + case "m": + return MILLI; + case "": + return BASE; + case "k": + return KILO; + case "M": + return MEGA; + case "G": + return GIGA; + case "T": + return TERA; + case "P": + return PETA; + case "Ki": + return KILO_BINARY; + case "Mi": + return MEGA_BINARY; + case "Gi": + return GIGA_BINARY; + case "Ti": + return TERA_BINARY; + case "Pi": + return PETA_BINARY; + default: + throw new IllegalArgumentException( + "Unknown unit '" + unit + "'. Known units are " + KNOWN_UNITS); + } + } + + /** + * Converts a value from one unit to another. Supported units can be obtained + * by inspecting the KNOWN_UNITS set. + * + * @param fromUnit the unit of the from value + * @param toUnit the target unit + * @param fromValue the value you wish to convert + * @return the value in toUnit + */ + public static Long convert(String fromUnit, String toUnit, Long fromValue) { + if (toUnit == null || fromUnit == null || fromValue == null) { + throw new IllegalArgumentException("One or more arguments are null"); + } + String overflowMsg = + "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit + + "' will result in an overflow of Long"; + if (fromUnit.equals(toUnit)) { + return fromValue; + } + Converter fc = getConverter(fromUnit); + Converter tc = getConverter(toUnit); + Long numerator = fc.numerator * tc.denominator; + Long denominator = fc.denominator * tc.numerator; + Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator; + if (numerator < denominator) { + if (numeratorMultiplierLimit < fromValue) { + throw new IllegalArgumentException(overflowMsg); + } + return (fromValue * numerator) / denominator; + } + if (numeratorMultiplierLimit > fromValue) { + return (numerator * fromValue) / denominator; + } + Long tmp = numerator / denominator; + if ((Long.MAX_VALUE / tmp) < fromValue) { + throw new IllegalArgumentException(overflowMsg); + } + return fromValue * tmp; + } + + /** + * Compare a value in a given unit with a value in another unit. The return + * value is equivalent to the value returned by compareTo. + * + * @param unitA first unit + * @param valueA first value + * @param unitB second unit + * @param valueB second value + * @return +1, 0 or -1 depending on whether the relationship is greater than, + * equal to or lesser than + */ + public static int compare(String unitA, Long valueA, String unitB, + Long valueB) { + if (unitA == null || unitB == null || !KNOWN_UNITS.contains(unitA) + || !KNOWN_UNITS.contains(unitB)) { + throw new IllegalArgumentException("Units cannot be null"); + } + if (!KNOWN_UNITS.contains(unitA)) { + throw new IllegalArgumentException("Unknown unit '" + unitA + "'"); + } + if (!KNOWN_UNITS.contains(unitB)) { + throw new IllegalArgumentException("Unknown unit '" + unitB + "'"); + } + Converter unitAC = getConverter(unitA); + Converter unitBC = getConverter(unitB); + if (unitA.equals(unitB)) { + return valueA.compareTo(valueB); + } + int unitAPos = SORTED_UNITS.indexOf(unitA); + int unitBPos = SORTED_UNITS.indexOf(unitB); + try { + Long tmpA = valueA; + Long tmpB = valueB; + if (unitAPos < unitBPos) { + tmpB = convert(unitB, unitA, valueB); + } else { + tmpA = convert(unitA, unitB, valueA); + } + return tmpA.compareTo(tmpB); + } catch (IllegalArgumentException ie) { + BigInteger tmpA = BigInteger.valueOf(valueA); + BigInteger tmpB = BigInteger.valueOf(valueB); + if (unitAPos < unitBPos) { + tmpB = tmpB.multiply(BigInteger.valueOf(unitBC.numerator)); + tmpB = tmpB.multiply(BigInteger.valueOf(unitAC.denominator)); + tmpB = tmpB.divide(BigInteger.valueOf(unitBC.denominator)); + tmpB = tmpB.divide(BigInteger.valueOf(unitAC.numerator)); + } else { + tmpA = tmpA.multiply(BigInteger.valueOf(unitAC.numerator)); + tmpA = tmpA.multiply(BigInteger.valueOf(unitBC.denominator)); + tmpA = tmpA.divide(BigInteger.valueOf(unitAC.denominator)); + tmpA = tmpA.divide(BigInteger.valueOf(unitBC.numerator)); + } + return tmpA.compareTo(tmpB); + } + } +} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index aa60c9c..a05c87a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -84,6 +84,7 @@ final ResourceCalculator resourceCalculator; Set accessibleLabels; + Set resourceTypes; final RMNodeLabelsManager labelManager; String defaultLabelExpression; @@ -99,6 +100,9 @@ // etc. QueueCapacities queueCapacities; + Map minResourceInformation = new HashMap<>(); + Map maxResourceInformation = new HashMap<>(); + private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); protected CapacitySchedulerContext csContext; @@ -275,6 +279,50 @@ void setupQueueConfigs(Resource clusterResource) this.defaultLabelExpression = csContext.getConfiguration().getDefaultNodeLabelExpression( getQueuePath()); + this.resourceTypes = new HashSet(); + for (String str : CapacitySchedulerConfiguration.DEFAULT_RESOURCE_TYPES + .split(",")) { + if (!str.trim().isEmpty()) { + resourceTypes.add(str.trim()); + } + } + + for (String resourceType : resourceTypes) { + ResourceInformation minResInfo = csContext.getConfiguration() + .getMinimumResourceRequirement(resourceType, getQueuePath()); + ResourceInformation maxResInfo = csContext.getConfiguration() + .getMaximumResourceRequirement(resourceType, getQueuePath()); + + // If min resource for a resource type is greater than its max resource, + // throw exception to handle such error configs. + if (minResInfo.compareTo(maxResInfo) > 0) { + throw new IllegalArgumentException("For '" + resourceType + + "', min resource configuration " + minResInfo + + " is greater than its max value:" + maxResInfo); + } + + // If parent's max resource is lesser to a specific child's max + // resource, throw exception to handle such error configs. + ResourceInformation parentMaxResource = parent + .getMaximumResourceInformation().get(resourceType); + if (parent.getMaximumResourceInformation().get(resourceType) + .compareTo(maxResInfo) > 0) { + throw new IllegalArgumentException("For '" + resourceType + + "', max resource configuration " + maxResInfo + + " is greater than parents max value:" + parentMaxResource); + } + + // If child's max resource is not set, but its parent max resource is + // set, we must set child max resource to its parent's. + if(maxResInfo == null && parentMaxResource != null) { + maxResInfo = parentMaxResource; + } + + if (minResInfo != null && maxResInfo != null) { + minResourceInformation.putIfAbsent(resourceType, minResInfo); + maxResourceInformation.putIfAbsent(resourceType, maxResInfo); + } + } // inherit from parent if labels not set if (this.accessibleLabels == null && parent != null) { @@ -941,4 +989,14 @@ protected void appFinished() { public Priority getPriority() { return this.priority; } + + @Override + public Map getMinimumResourceInformation() { + return minResourceInformation; + } + + @Override + public Map getMaximumResourceInformation() { + return maxResourceInformation; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 6d30386..59ba589 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -364,4 +365,18 @@ public void validateSubmitApplication(ApplicationId applicationId, * @return queue priority */ Priority getPriority(); + + /** + * Get minimum resource requirement for a queue. + * + * @return queue's minimum resource + */ + Map getMinimumResourceInformation(); + + /** + * Get maximum resource requirement for a queue. + * + * @return queue's maximum resource + */ + Map getMaximumResourceInformation(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 43ec390..6aa843d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -285,6 +285,14 @@ @Private public static final boolean DEFAULT_LAZY_PREEMPTION_ENABLED = false; + /** Configuring absolute min/max resources in a queue **/ + @Private + public static final String MINIMUM_RESOURCE = "min-resource"; + + public static final String MAXIMUM_RESOURCE = "max-resource"; + + public static final String DEFAULT_RESOURCE_TYPES = "memory,vcores"; + AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser(); public CapacitySchedulerConfiguration() { @@ -1392,4 +1400,59 @@ public void setPUOrderingPolicyUnderUtilizedPreemptionMoveReservation( QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY, UNDER_UTILIZED_PREEMPTION_MOVE_RESERVATION), allowMoveReservation); } + + public static String getUnits(String resourceValue) { + String units; + for (int i = 0; i < resourceValue.length(); i++) { + if (Character.isAlphabetic(resourceValue.charAt(i))) { + units = resourceValue.substring(i); + if (StringUtils.isAlpha(units)) { + return units; + } + } + } + return ""; + } + + /** + * Get absolute minimum resource from a queue. + * + * @param type resource type + * @param queue queue path + * @return ResourceInformation + */ + public ResourceInformation getMinimumResourceRequirement(String type, + String queue) { + return createResourceInformation(type, queue, MINIMUM_RESOURCE); + } + + /** + * Get absolute maximum resource from a queue. + * + * @param type resource type + * @param queue queue path + * @return ResourceInformation + */ + public ResourceInformation getMaximumResourceRequirement(String type, + String queue) { + return createResourceInformation(type, queue, MAXIMUM_RESOURCE); + } + + private ResourceInformation createResourceInformation(String type, + String queue, String resourceType) { + String maxResource = get( + getQueuePrefix(queue) + type + DOT + resourceType); + if (maxResource == null || maxResource.isEmpty()) { + return null; + } + + String units = getUnits(maxResource); + Long resourceValue = Long.valueOf( + maxResource.substring(0, maxResource.length() - units.length())); + ResourceInformation resourceInfo = ResourceInformation.newInstance(type, + units); + resourceInfo.setValue(resourceValue); + resourceInfo.setUnits(units); + return resourceInfo; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 1579472..5497cf5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -34,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.AccessType; @@ -45,7 +44,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger; @@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSetUtils; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.Resources; import java.io.IOException; @@ -163,9 +162,37 @@ void setChildQueues(Collection childQueues) { writeLock.lock(); // Validate float childCapacities = 0; + Map childQueueMinResource = new HashMap<>(); + Map childQueueMaxResource = new HashMap<>(); for (CSQueue queue : childQueues) { childCapacities += queue.getCapacity(); + for (String resType : this.resourceTypes) { + if (!childQueueMinResource.containsKey(resType)) { + childQueueMinResource.put(resType, 0l); + } + Long minRes = childQueueMinResource.get(resType); + minRes += getUnitConvertedResource( + queue.getMinimumResourceInformation().get(resType), resType); + + if (!childQueueMaxResource.containsKey(resType)) { + childQueueMaxResource.put(resType, 0l); + } + Long maxRes = childQueueMaxResource.get(resType); + maxRes += getUnitConvertedResource( + queue.getMaximumResourceInformation().get(resType), resType); + } } + + // Ensure that for each parent queue: parent.min-resource >= Σ(child.min-resource). + for (String resType : this.resourceTypes) { + Long minRes = getUnitConvertedResource( + this.getMinimumResourceInformation().get(resType), resType); + if (minRes < childQueueMinResource.get(resType)) { + throw new IllegalArgumentException("Parent Queues" + + " capacity is less than" + " to its children " + queueName); + } + } + float delta = Math.abs(1.0f - childCapacities); // crude way to check // allow capacities being set to 0, and enforce child 0 if parent is 0 if (((queueCapacities.getCapacity() > 0) && (delta > PRECISION)) || ( @@ -200,6 +227,15 @@ void setChildQueues(Collection childQueues) { } } + private long getUnitConvertedResource(ResourceInformation resource, + String resType) { + if (resource == null) { + return 0l; + } + return UnitsConversionUtil.convert(resource.getUnits(), "", + resource.getValue()); + } + @Override public String getQueuePath() { String parentPath = ((parent == null) ? "" : (parent.getQueuePath() + ".")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java index cc4af3d..d6a28cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java @@ -46,7 +46,7 @@ public QueueCapacities(boolean isRoot) { capacitiesMap = new HashMap(); this.isRoot = isRoot; } - + // Usage enum here to make implement cleaner private enum CapacityType { USED_CAP(0), ABS_USED_CAP(1), MAX_CAP(2), ABS_MAX_CAP(3), CAP(4), ABS_CAP(5), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ResourceInformation.java new file mode 100644 index 0000000..893d5f2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ResourceInformation.java @@ -0,0 +1,235 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.util.UnitsConversionUtil; + +/** + * Class to encapsulate information about a Resource - the name of the resource, + * the units(milli, micro, etc), the type(countable), and the value. + */ +public class ResourceInformation implements Comparable { + + private String name; + private String units; + private Long value; + private Long minimumAllocation; + private Long maximumAllocation; + + private static final String MEMORY_URI = "memory-mb"; + private static final String VCORES_URI = "vcores"; + + public static final ResourceInformation MEMORY_MB = ResourceInformation + .newInstance(MEMORY_URI, "Mi"); + public static final ResourceInformation VCORES = ResourceInformation + .newInstance(VCORES_URI); + + /** + * Get the name for the resource. + * + * @return resource name + */ + public String getName() { + return name; + } + + /** + * Set the name for the resource. + * + * @param rName + * name for the resource + */ + public void setName(String rName) { + this.name = rName; + } + + /** + * Get units for the resource. + * + * @return units for the resource + */ + public String getUnits() { + return units; + } + + /** + * Set the units for the resource. + * + * @param rUnits + * units for the resource + */ + public void setUnits(String rUnits) { + if (!UnitsConversionUtil.KNOWN_UNITS.contains(rUnits)) { + throw new IllegalArgumentException("Unknown unit '" + rUnits + + "'. Known units are " + UnitsConversionUtil.KNOWN_UNITS); + } + this.units = rUnits; + } + + /** + * Get the value for the resource. + * + * @return the resource value + */ + public Long getValue() { + return value; + } + + /** + * Set the value for the resource. + * + * @param rValue + * the resource value + */ + public void setValue(Long rValue) { + this.value = rValue; + } + + /** + * Get the minimum allocation for the resource. + * + * @return the minimum allocation for the resource + */ + public Long getMinimumAllocation() { + return minimumAllocation; + } + + /** + * Set the minimum allocation for the resource. + * + * @param minimumAllocation + * the minimum allocation for the resource + */ + public void setMinimumAllocation(Long minimumAllocation) { + this.minimumAllocation = minimumAllocation; + } + + /** + * Get the maximum allocation for the resource. + * + * @return the maximum allocation for the resource + */ + public Long getMaximumAllocation() { + return maximumAllocation; + } + + /** + * Set the maximum allocation for the resource. + * + * @param maximumAllocation + * the maximum allocation for the resource + */ + public void setMaximumAllocation(Long maximumAllocation) { + this.maximumAllocation = maximumAllocation; + } + + /** + * Create a new instance of ResourceInformation from another object. + * + * @param other + * the object from which the new object should be created + * @return the new ResourceInformation object + */ + public static ResourceInformation newInstance(ResourceInformation other) { + ResourceInformation ret = new ResourceInformation(); + ret.setName(other.getName()); + ret.setUnits(other.getUnits()); + ret.setValue(other.getValue()); + ret.setMinimumAllocation(other.getMinimumAllocation()); + ret.setMaximumAllocation(other.getMaximumAllocation()); + return ret; + } + + public static ResourceInformation newInstance(String name, String units, + Long value, Long minimumAllocation, Long maximumAllocation) { + ResourceInformation ret = new ResourceInformation(); + ret.setName(name); + ret.setUnits(units); + ret.setValue(value); + ret.setMinimumAllocation(minimumAllocation); + ret.setMaximumAllocation(maximumAllocation); + return ret; + } + + public static ResourceInformation newInstance(String name, String units, + Long value) { + return ResourceInformation.newInstance(name, units, value, 0L, + Long.MAX_VALUE); + } + + public static ResourceInformation newInstance(String name, String units) { + return ResourceInformation.newInstance(name, units, 0L, 0L, Long.MAX_VALUE); + } + + public static ResourceInformation newInstance(String name, Long value) { + return ResourceInformation.newInstance(name, "", value, 0L, Long.MAX_VALUE); + } + + public static ResourceInformation newInstance(String name) { + return ResourceInformation.newInstance(name, ""); + } + + @Override + public String toString() { + return "name: " + this.name + ", units: " + this.units + ", value: " + value + + ", minimum allocation: " + minimumAllocation + + ", maximum allocation: " + maximumAllocation; + } + + public String getShorthandRepresentation() { + return "" + this.value + this.units; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof ResourceInformation)) { + return false; + } + ResourceInformation r = (ResourceInformation) obj; + int cmp = UnitsConversionUtil.compare(this.units, this.value, r.units, + r.value); + return this.name.equals(r.getName()) && (cmp == 0); + } + + @Override + public int hashCode() { + final int prime = 263167; + int result = 939769357 + name.hashCode(); // prime * result = 939769357 + // initially + result = prime * result + units.hashCode(); + result = prime * result + value.hashCode(); + return result; + } + + @Override + public int compareTo(ResourceInformation other) { + int diff = this.name.compareTo(other.name); + if (diff == 0) { + diff = UnitsConversionUtil.compare(this.units, this.value, other.units, + other.value); + } + return diff; + } +} -- 2.10.1 (Apple Git-78)